feat: list_bases returns registered table storage prefixes

Return the additional storage bases for the current table snapshot
on native, memory, namespace, and Cloud clients.
This commit is contained in:
geruh
2026-08-20 18:46:07 -07:00
parent 047f431837
commit cfcbcfbc92
14 changed files with 418 additions and 43 deletions
+1
View File
@@ -378,6 +378,7 @@ class Table:
def take_row_ids(self, row_ids: list[int]) -> TakeQuery: ...
async def blob_columns(self) -> list[str]: ...
async def add_bases(self, bases: list[Any]) -> None: ...
async def list_bases(self) -> list[tuple[str, Optional[str], bool]]: ...
async def fetch_blobs(
self, column: str, row_ids: list[int]
) -> pa.LargeBinaryArray: ...
+4
View File
@@ -1089,6 +1089,10 @@ class RemoteTable(Table):
"""Register additional storage bases for this table."""
LOOP.run(self._table.add_bases(bases))
def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
return LOOP.run(self._table.list_bases())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
+15
View File
@@ -1596,6 +1596,10 @@ class Table(ABC):
"""
raise NotImplementedError
def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
raise NotImplementedError
@abstractmethod
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
@@ -2448,6 +2452,10 @@ class LanceTable(Table):
) -> None:
LOOP.run(self._table.add_bases(bases))
def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
return LOOP.run(self._table.list_bases())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
@@ -6312,6 +6320,13 @@ class AsyncTable:
"""
await self._inner.add_bases(_normalize_bases(bases))
async def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
return [
TableBase(path=path, name=name, is_dataset_root=is_dataset_root)
for path, name, is_dataset_root in await self._inner.list_bases()
]
async def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
+37 -12
View File
@@ -7,7 +7,33 @@ import pytest
import lancedb
def test_add_bases_accepts_named_and_dataset_root(tmp_path):
def test_list_bases_reflects_added_bases(tmp_path):
media = tmp_path / "media"
media.mkdir()
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
assert table.list_bases() == []
table.add_bases(media.as_uri())
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
def test_add_bases_accepts_two_unnamed_paths(tmp_path):
media = tmp_path / "media"
other = tmp_path / "other"
media.mkdir()
other.mkdir()
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
table.add_bases([media.as_uri(), other.as_uri()])
assert table.list_bases() == [
lancedb.TableBase(path=media.as_uri()),
lancedb.TableBase(path=other.as_uri()),
]
def test_add_bases_records_name_and_dataset_root(tmp_path):
media = tmp_path / "media"
parent = tmp_path / "parent"
media.mkdir()
@@ -23,17 +49,10 @@ def test_add_bases_accepts_named_and_dataset_root(tmp_path):
),
]
)
def test_add_bases_accepts_two_unnamed_paths(tmp_path):
media = tmp_path / "media"
other = tmp_path / "other"
media.mkdir()
other.mkdir()
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
table.add_bases([media.as_uri(), other.as_uri()])
assert table.list_bases() == [
lancedb.TableBase(path=media.as_uri(), name="media", is_dataset_root=False),
lancedb.TableBase(path=parent.as_uri(), name="parent", is_dataset_root=True),
]
def test_add_bases_rejects_dict_input(tmp_path):
@@ -51,7 +70,9 @@ async def test_async_add_bases_accepts_file_uri(tmp_path):
db = await lancedb.connect_async(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = await db.create_table("photos", schema=schema)
assert await table.list_bases() == []
await table.add_bases(media.as_uri())
assert await table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
def test_memory_add_bases_accepts_file_uri(tmp_path):
@@ -60,7 +81,9 @@ def test_memory_add_bases_accepts_file_uri(tmp_path):
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
assert table.list_bases() == []
table.add_bases(media.as_uri())
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
def test_namespace_add_bases_accepts_file_uri(tmp_path):
@@ -69,4 +92,6 @@ def test_namespace_add_bases_accepts_file_uri(tmp_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path / "ns")})
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
assert table.list_bases() == []
table.add_bases(media.as_uri())
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
+44 -1
View File
@@ -2308,7 +2308,7 @@ def test_remote_connection_jobs_surface():
job.wait(timeout=timedelta(seconds=5))
def test_remote_add_bases_posts_the_bases_array():
def test_remote_add_and_list_bases():
captured_body = {}
def handler(request):
@@ -2324,6 +2324,13 @@ def test_remote_add_bases_posts_the_bases_array():
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 2}')
elif request.path == "/v1/table/test/bases/list/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
b'{"bases":[{"path":"s3://bucket/media/","isDatasetRoot":false}]}'
)
else:
request.send_response(404)
request.end_headers()
@@ -2331,6 +2338,13 @@ def test_remote_add_bases_posts_the_bases_array():
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
table.add_bases(lancedb.TableBase(path="s3://bucket/media/"))
assert table.list_bases() == [
lancedb.TableBase(
path="s3://bucket/media/",
name=None,
is_dataset_root=False,
)
]
assert captured_body["bases"] == [
{
@@ -2339,3 +2353,32 @@ def test_remote_add_bases_posts_the_bases_array():
}
]
def test_remote_list_bases_returns_named_dataset_root():
def handler(request):
if request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
elif request.path == "/v1/table/test/bases/list/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
b'{"bases":[{"path":"s3://bucket/archive/","name":"archive","isDatasetRoot":true}]}'
)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
assert table.list_bases() == [
lancedb.TableBase(
path="s3://bucket/archive/",
name="archive",
is_dataset_root=True,
)
]
+11
View File
@@ -1265,6 +1265,17 @@ impl Table {
})
}
pub fn list_bases(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let bases = inner.list_bases().await.infer_error()?;
Ok(bases
.into_iter()
.map(|base| (base.path, base.name, base.is_dataset_root))
.collect::<Vec<_>>())
})
}
/// Read blob bytes for `row_ids` from blob v2 column `column`.
#[pyo3(signature = (column, row_ids))]
pub fn fetch_blobs(