mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-30 09:58:20 +00:00
feat: list_bases returns registered table storage prefixes
Return the additional storage bases for the current table snapshot on native, memory, namespace, and Cloud clients.
This commit is contained in:
@@ -378,6 +378,7 @@ class Table:
|
||||
def take_row_ids(self, row_ids: list[int]) -> TakeQuery: ...
|
||||
async def blob_columns(self) -> list[str]: ...
|
||||
async def add_bases(self, bases: list[Any]) -> None: ...
|
||||
async def list_bases(self) -> list[tuple[str, Optional[str], bool]]: ...
|
||||
async def fetch_blobs(
|
||||
self, column: str, row_ids: list[int]
|
||||
) -> pa.LargeBinaryArray: ...
|
||||
|
||||
@@ -1089,6 +1089,10 @@ class RemoteTable(Table):
|
||||
"""Register additional storage bases for this table."""
|
||||
LOOP.run(self._table.add_bases(bases))
|
||||
|
||||
def list_bases(self) -> list[TableBase]:
|
||||
"""Return the additional storage bases for the current table snapshot."""
|
||||
return LOOP.run(self._table.list_bases())
|
||||
|
||||
def fetch_blobs(
|
||||
self, column: str, row_ids: Union[list[int], pa.Table]
|
||||
) -> pa.LargeBinaryArray:
|
||||
|
||||
@@ -1596,6 +1596,10 @@ class Table(ABC):
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def list_bases(self) -> list[TableBase]:
|
||||
"""Return the additional storage bases for the current table snapshot."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def fetch_blobs(
|
||||
self, column: str, row_ids: Union[list[int], pa.Table]
|
||||
@@ -2448,6 +2452,10 @@ class LanceTable(Table):
|
||||
) -> None:
|
||||
LOOP.run(self._table.add_bases(bases))
|
||||
|
||||
def list_bases(self) -> list[TableBase]:
|
||||
"""Return the additional storage bases for the current table snapshot."""
|
||||
return LOOP.run(self._table.list_bases())
|
||||
|
||||
def fetch_blobs(
|
||||
self, column: str, row_ids: Union[list[int], pa.Table]
|
||||
) -> pa.LargeBinaryArray:
|
||||
@@ -6312,6 +6320,13 @@ class AsyncTable:
|
||||
"""
|
||||
await self._inner.add_bases(_normalize_bases(bases))
|
||||
|
||||
async def list_bases(self) -> list[TableBase]:
|
||||
"""Return the additional storage bases for the current table snapshot."""
|
||||
return [
|
||||
TableBase(path=path, name=name, is_dataset_root=is_dataset_root)
|
||||
for path, name, is_dataset_root in await self._inner.list_bases()
|
||||
]
|
||||
|
||||
async def fetch_blobs(
|
||||
self, column: str, row_ids: Union[list[int], pa.Table]
|
||||
) -> pa.LargeBinaryArray:
|
||||
|
||||
@@ -7,7 +7,33 @@ import pytest
|
||||
import lancedb
|
||||
|
||||
|
||||
def test_add_bases_accepts_named_and_dataset_root(tmp_path):
|
||||
def test_list_bases_reflects_added_bases(tmp_path):
|
||||
media = tmp_path / "media"
|
||||
media.mkdir()
|
||||
db = lancedb.connect(tmp_path / "db")
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = db.create_table("photos", schema=schema)
|
||||
assert table.list_bases() == []
|
||||
table.add_bases(media.as_uri())
|
||||
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
|
||||
|
||||
|
||||
def test_add_bases_accepts_two_unnamed_paths(tmp_path):
|
||||
media = tmp_path / "media"
|
||||
other = tmp_path / "other"
|
||||
media.mkdir()
|
||||
other.mkdir()
|
||||
db = lancedb.connect(tmp_path / "db")
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = db.create_table("photos", schema=schema)
|
||||
table.add_bases([media.as_uri(), other.as_uri()])
|
||||
assert table.list_bases() == [
|
||||
lancedb.TableBase(path=media.as_uri()),
|
||||
lancedb.TableBase(path=other.as_uri()),
|
||||
]
|
||||
|
||||
|
||||
def test_add_bases_records_name_and_dataset_root(tmp_path):
|
||||
media = tmp_path / "media"
|
||||
parent = tmp_path / "parent"
|
||||
media.mkdir()
|
||||
@@ -23,17 +49,10 @@ def test_add_bases_accepts_named_and_dataset_root(tmp_path):
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_add_bases_accepts_two_unnamed_paths(tmp_path):
|
||||
media = tmp_path / "media"
|
||||
other = tmp_path / "other"
|
||||
media.mkdir()
|
||||
other.mkdir()
|
||||
db = lancedb.connect(tmp_path / "db")
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = db.create_table("photos", schema=schema)
|
||||
table.add_bases([media.as_uri(), other.as_uri()])
|
||||
assert table.list_bases() == [
|
||||
lancedb.TableBase(path=media.as_uri(), name="media", is_dataset_root=False),
|
||||
lancedb.TableBase(path=parent.as_uri(), name="parent", is_dataset_root=True),
|
||||
]
|
||||
|
||||
|
||||
def test_add_bases_rejects_dict_input(tmp_path):
|
||||
@@ -51,7 +70,9 @@ async def test_async_add_bases_accepts_file_uri(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path / "db")
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = await db.create_table("photos", schema=schema)
|
||||
assert await table.list_bases() == []
|
||||
await table.add_bases(media.as_uri())
|
||||
assert await table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
|
||||
|
||||
|
||||
def test_memory_add_bases_accepts_file_uri(tmp_path):
|
||||
@@ -60,7 +81,9 @@ def test_memory_add_bases_accepts_file_uri(tmp_path):
|
||||
db = lancedb.connect("memory:///")
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = db.create_table("photos", schema=schema)
|
||||
assert table.list_bases() == []
|
||||
table.add_bases(media.as_uri())
|
||||
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
|
||||
|
||||
|
||||
def test_namespace_add_bases_accepts_file_uri(tmp_path):
|
||||
@@ -69,4 +92,6 @@ def test_namespace_add_bases_accepts_file_uri(tmp_path):
|
||||
db = lancedb.connect_namespace("dir", {"root": str(tmp_path / "ns")})
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
table = db.create_table("photos", schema=schema)
|
||||
assert table.list_bases() == []
|
||||
table.add_bases(media.as_uri())
|
||||
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
|
||||
|
||||
@@ -2308,7 +2308,7 @@ def test_remote_connection_jobs_surface():
|
||||
job.wait(timeout=timedelta(seconds=5))
|
||||
|
||||
|
||||
def test_remote_add_bases_posts_the_bases_array():
|
||||
def test_remote_add_and_list_bases():
|
||||
captured_body = {}
|
||||
|
||||
def handler(request):
|
||||
@@ -2324,6 +2324,13 @@ def test_remote_add_bases_posts_the_bases_array():
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(b'{"version": 2}')
|
||||
elif request.path == "/v1/table/test/bases/list/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
b'{"bases":[{"path":"s3://bucket/media/","isDatasetRoot":false}]}'
|
||||
)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
@@ -2331,6 +2338,13 @@ def test_remote_add_bases_posts_the_bases_array():
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
table = db.open_table("test")
|
||||
table.add_bases(lancedb.TableBase(path="s3://bucket/media/"))
|
||||
assert table.list_bases() == [
|
||||
lancedb.TableBase(
|
||||
path="s3://bucket/media/",
|
||||
name=None,
|
||||
is_dataset_root=False,
|
||||
)
|
||||
]
|
||||
|
||||
assert captured_body["bases"] == [
|
||||
{
|
||||
@@ -2339,3 +2353,32 @@ def test_remote_add_bases_posts_the_bases_array():
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_remote_list_bases_returns_named_dataset_root():
|
||||
def handler(request):
|
||||
if request.path == "/v1/table/test/describe/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
|
||||
elif request.path == "/v1/table/test/bases/list/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(
|
||||
b'{"bases":[{"path":"s3://bucket/archive/","name":"archive","isDatasetRoot":true}]}'
|
||||
)
|
||||
else:
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
table = db.open_table("test")
|
||||
assert table.list_bases() == [
|
||||
lancedb.TableBase(
|
||||
path="s3://bucket/archive/",
|
||||
name="archive",
|
||||
is_dataset_root=True,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@@ -1265,6 +1265,17 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn list_bases(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let bases = inner.list_bases().await.infer_error()?;
|
||||
Ok(bases
|
||||
.into_iter()
|
||||
.map(|base| (base.path, base.name, base.is_dataset_root))
|
||||
.collect::<Vec<_>>())
|
||||
})
|
||||
}
|
||||
|
||||
/// Read blob bytes for `row_ids` from blob v2 column `column`.
|
||||
#[pyo3(signature = (column, row_ids))]
|
||||
pub fn fetch_blobs(
|
||||
|
||||
Reference in New Issue
Block a user