Compare commits

..

1 Commits

Author SHA1 Message Date
geruh cfcbcfbc92 feat: list_bases returns registered table storage prefixes
Return the additional storage bases for the current table snapshot
on native, memory, namespace, and Cloud clients.
2026-08-21 00:48:38 -07:00
14 changed files with 418 additions and 43 deletions
+62
View File
@@ -1028,11 +1028,30 @@ describe("remote connection jobs surface", () => {
});
return;
}
if (path.endsWith("/bases/list/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
bases: [
{
path: "s3://bucket/media/",
isDatasetRoot: false,
},
],
}),
);
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("photos");
await table.addBases({ path: "s3://bucket/media/" });
expect(await table.listBases()).toEqual([
{
path: "s3://bucket/media/",
isDatasetRoot: false,
},
]);
},
);
expect(postedBodies).toEqual([
@@ -1046,4 +1065,47 @@ describe("remote connection jobs surface", () => {
},
]);
});
it("listBases returns a named dataset-root base", async () => {
await withMockDatabase(
(req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "photos",
version: 1,
schema: { fields: [] },
}),
);
return;
}
if (path.endsWith("/bases/list/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
bases: [
{
path: "s3://bucket/archive/",
name: "archive",
isDatasetRoot: true,
},
],
}),
);
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("photos");
expect(await table.listBases()).toEqual([
{
path: "s3://bucket/archive/",
name: "archive",
isDatasetRoot: true,
},
]);
},
);
});
});
+8 -2
View File
@@ -3413,7 +3413,7 @@ describe("table bases", () => {
});
afterEach(() => tmpDir.removeCallback());
it("addBases accepts a file uri", async () => {
it("listBases reflects added bases", async () => {
const conn = await connect(tmpDir.name);
const table = await conn.createEmptyTable(
"photos",
@@ -3421,6 +3421,12 @@ describe("table bases", () => {
);
const media = path.join(tmpDir.name, "media");
fs.mkdirSync(media);
await table.addBases(pathToFileURL(media).toString());
const location = pathToFileURL(media).toString();
expect(await table.listBases()).toEqual([]);
await table.addBases(location);
expect(await table.listBases()).toEqual([
{ path: location, isDatasetRoot: false },
]);
});
});
+7
View File
@@ -591,6 +591,9 @@ export abstract class Table {
bases: string | TableBase | Array<string | TableBase>,
): Promise<void>;
/** Return the additional storage bases for the current table snapshot. */
abstract listBases(): Promise<TableBase[]>;
/**
* Fill the rows of a computed column that hold no value yet.
*
@@ -1230,6 +1233,10 @@ export class LocalTable extends Table {
await this.inner.addBases(normalizeBases(bases));
}
async listBases(): Promise<TableBase[]> {
return await this.inner.listBases();
}
async refreshColumn(column: string): Promise<RefreshColumnResult> {
return await this.inner.refreshColumn(column);
}
+22
View File
@@ -459,6 +459,18 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn list_bases(&self) -> napi::Result<Vec<TableBase>> {
Ok(self
.inner_ref()?
.list_bases()
.await
.default_error()?
.into_iter()
.map(TableBase::from)
.collect())
}
#[napi(catch_unwind)]
pub async fn drop_columns(&self, columns: Vec<String>) -> napi::Result<DropColumnsResult> {
let col_refs = columns.iter().map(String::as_str).collect::<Vec<_>>();
@@ -725,6 +737,16 @@ pub struct TableBase {
pub is_dataset_root: bool,
}
impl From<LanceTableBase> for TableBase {
fn from(base: LanceTableBase) -> Self {
Self {
path: base.path,
name: base.name,
is_dataset_root: base.is_dataset_root,
}
}
}
#[napi(object)]
/// A description of an index currently configured on a column
pub struct IndexConfig {
+1
View File
@@ -378,6 +378,7 @@ class Table:
def take_row_ids(self, row_ids: list[int]) -> TakeQuery: ...
async def blob_columns(self) -> list[str]: ...
async def add_bases(self, bases: list[Any]) -> None: ...
async def list_bases(self) -> list[tuple[str, Optional[str], bool]]: ...
async def fetch_blobs(
self, column: str, row_ids: list[int]
) -> pa.LargeBinaryArray: ...
+4
View File
@@ -1089,6 +1089,10 @@ class RemoteTable(Table):
"""Register additional storage bases for this table."""
LOOP.run(self._table.add_bases(bases))
def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
return LOOP.run(self._table.list_bases())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
+15
View File
@@ -1596,6 +1596,10 @@ class Table(ABC):
"""
raise NotImplementedError
def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
raise NotImplementedError
@abstractmethod
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
@@ -2448,6 +2452,10 @@ class LanceTable(Table):
) -> None:
LOOP.run(self._table.add_bases(bases))
def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
return LOOP.run(self._table.list_bases())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
@@ -6312,6 +6320,13 @@ class AsyncTable:
"""
await self._inner.add_bases(_normalize_bases(bases))
async def list_bases(self) -> list[TableBase]:
"""Return the additional storage bases for the current table snapshot."""
return [
TableBase(path=path, name=name, is_dataset_root=is_dataset_root)
for path, name, is_dataset_root in await self._inner.list_bases()
]
async def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
) -> pa.LargeBinaryArray:
+37 -12
View File
@@ -7,7 +7,33 @@ import pytest
import lancedb
def test_add_bases_accepts_named_and_dataset_root(tmp_path):
def test_list_bases_reflects_added_bases(tmp_path):
media = tmp_path / "media"
media.mkdir()
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
assert table.list_bases() == []
table.add_bases(media.as_uri())
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
def test_add_bases_accepts_two_unnamed_paths(tmp_path):
media = tmp_path / "media"
other = tmp_path / "other"
media.mkdir()
other.mkdir()
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
table.add_bases([media.as_uri(), other.as_uri()])
assert table.list_bases() == [
lancedb.TableBase(path=media.as_uri()),
lancedb.TableBase(path=other.as_uri()),
]
def test_add_bases_records_name_and_dataset_root(tmp_path):
media = tmp_path / "media"
parent = tmp_path / "parent"
media.mkdir()
@@ -23,17 +49,10 @@ def test_add_bases_accepts_named_and_dataset_root(tmp_path):
),
]
)
def test_add_bases_accepts_two_unnamed_paths(tmp_path):
media = tmp_path / "media"
other = tmp_path / "other"
media.mkdir()
other.mkdir()
db = lancedb.connect(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
table.add_bases([media.as_uri(), other.as_uri()])
assert table.list_bases() == [
lancedb.TableBase(path=media.as_uri(), name="media", is_dataset_root=False),
lancedb.TableBase(path=parent.as_uri(), name="parent", is_dataset_root=True),
]
def test_add_bases_rejects_dict_input(tmp_path):
@@ -51,7 +70,9 @@ async def test_async_add_bases_accepts_file_uri(tmp_path):
db = await lancedb.connect_async(tmp_path / "db")
schema = pa.schema([pa.field("id", pa.int64())])
table = await db.create_table("photos", schema=schema)
assert await table.list_bases() == []
await table.add_bases(media.as_uri())
assert await table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
def test_memory_add_bases_accepts_file_uri(tmp_path):
@@ -60,7 +81,9 @@ def test_memory_add_bases_accepts_file_uri(tmp_path):
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
assert table.list_bases() == []
table.add_bases(media.as_uri())
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
def test_namespace_add_bases_accepts_file_uri(tmp_path):
@@ -69,4 +92,6 @@ def test_namespace_add_bases_accepts_file_uri(tmp_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path / "ns")})
schema = pa.schema([pa.field("id", pa.int64())])
table = db.create_table("photos", schema=schema)
assert table.list_bases() == []
table.add_bases(media.as_uri())
assert table.list_bases() == [lancedb.TableBase(path=media.as_uri())]
+44 -1
View File
@@ -2308,7 +2308,7 @@ def test_remote_connection_jobs_surface():
job.wait(timeout=timedelta(seconds=5))
def test_remote_add_bases_posts_the_bases_array():
def test_remote_add_and_list_bases():
captured_body = {}
def handler(request):
@@ -2324,6 +2324,13 @@ def test_remote_add_bases_posts_the_bases_array():
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 2}')
elif request.path == "/v1/table/test/bases/list/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
b'{"bases":[{"path":"s3://bucket/media/","isDatasetRoot":false}]}'
)
else:
request.send_response(404)
request.end_headers()
@@ -2331,6 +2338,13 @@ def test_remote_add_bases_posts_the_bases_array():
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
table.add_bases(lancedb.TableBase(path="s3://bucket/media/"))
assert table.list_bases() == [
lancedb.TableBase(
path="s3://bucket/media/",
name=None,
is_dataset_root=False,
)
]
assert captured_body["bases"] == [
{
@@ -2339,3 +2353,32 @@ def test_remote_add_bases_posts_the_bases_array():
}
]
def test_remote_list_bases_returns_named_dataset_root():
def handler(request):
if request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
elif request.path == "/v1/table/test/bases/list/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
b'{"bases":[{"path":"s3://bucket/archive/","name":"archive","isDatasetRoot":true}]}'
)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
assert table.list_bases() == [
lancedb.TableBase(
path="s3://bucket/archive/",
name="archive",
is_dataset_root=True,
)
]
+11
View File
@@ -1265,6 +1265,17 @@ impl Table {
})
}
pub fn list_bases(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let bases = inner.list_bases().await.infer_error()?;
Ok(bases
.into_iter()
.map(|base| (base.path, base.name, base.is_dataset_root))
.collect::<Vec<_>>())
})
}
/// Read blob bytes for `row_ids` from blob v2 column `column`.
#[pyo3(signature = (column, row_ids))]
pub fn fetch_blobs(
+44
View File
@@ -2249,6 +2249,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
self.add_bases_impl(bases).await
}
async fn list_bases(&self) -> Result<Vec<crate::table::TableBase>> {
self.list_bases_impl().await
}
async fn fetch_blobs(&self, column: &str, row_ids: &[u64]) -> Result<LargeBinaryArray> {
self.fetch_blobs_impl(column, row_ids).await
}
@@ -4117,6 +4121,46 @@ mod tests {
table.add_bases(["s3://bucket/media/"]).await.unwrap();
}
#[tokio::test]
async fn list_bases_posts_and_returns_the_bases() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/bases/list/");
http::Response::builder()
.status(200)
.body(r#"{"bases":[{"path":"s3://bucket/media/","isDatasetRoot":false}]}"#)
.unwrap()
});
let bases = table.list_bases().await.unwrap();
assert_eq!(
bases,
vec![crate::table::TableBase::from("s3://bucket/media/")]
);
}
#[tokio::test]
async fn list_bases_returns_named_dataset_root_entries() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/bases/list/");
http::Response::builder()
.status(200)
.body(r#"{"bases":[{"path":"s3://bucket/archive/","name":"archive","isDatasetRoot":true}]}"#)
.unwrap()
});
let bases = table.list_bases().await.unwrap();
assert_eq!(
bases,
vec![crate::table::TableBase {
path: "s3://bucket/archive/".into(),
name: Some("archive".into()),
is_dataset_root: true,
}]
);
}
#[tokio::test]
async fn test_add_bases_rejects_empty_response() {
let table = Table::new_with_handler("my_table", |_request| {
+27 -1
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Cloud HTTP for registering extra table storage bases.
//! Cloud HTTP for registering and listing extra table storage bases.
use serde::Deserialize;
@@ -16,6 +16,11 @@ struct AddBasesResponse {
version: u64,
}
#[derive(Debug, Deserialize)]
struct ListBasesResponse {
bases: Vec<crate::table::TableBase>,
}
impl<S: HttpSend> RemoteTable<S> {
pub(super) async fn add_bases_impl(&self, bases: &[crate::table::TableBase]) -> Result<()> {
self.check_mutable().await?;
@@ -39,4 +44,25 @@ impl<S: HttpSend> RemoteTable<S> {
self.track_write_version(parsed.version);
Ok(())
}
pub(super) async fn list_bases_impl(&self) -> Result<Vec<crate::table::TableBase>> {
let version = self.current_version().await;
let mut body = serde_json::json!({ "version": version });
self.apply_branch_body(&mut body);
let request = self
.post_read(&format!("/v1/table/{}/bases/list/", self.identifier))
.json(&body);
let (request_id, response) = self.send(request, true).await?;
let response = self.check_table_response(&request_id, response).await?;
let body = response.text().await.err_to_http(request_id.clone())?;
let parsed: ListBasesResponse = serde_json::from_str(&body).map_err(|e| Error::Http {
source: format!(
"The server returned an invalid response while listing table bases: {e}"
)
.into(),
request_id,
status_code: None,
})?;
Ok(parsed.bases)
}
}
+25
View File
@@ -717,6 +717,12 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
message: "Registering table bases is not supported for this table type.".into(),
})
}
/// Return the additional storage bases for the current table snapshot.
async fn list_bases(&self) -> Result<Vec<TableBase>> {
Err(Error::NotSupported {
message: "Listing table bases is not supported for this table type.".into(),
})
}
/// Materialize blob bytes for the given row ids. See [`Table::fetch_blobs`].
async fn fetch_blobs(&self, _column: &str, _row_ids: &[u64]) -> Result<LargeBinaryArray> {
Err(Error::NotSupported {
@@ -1199,6 +1205,11 @@ impl Table {
self.inner.add_bases(&bases).await
}
/// Return the additional storage bases for the current table snapshot.
pub async fn list_bases(&self) -> Result<Vec<TableBase>> {
self.inner.list_bases().await
}
/// Materialize blob bytes for the given row ids.
///
/// Output matches `row_ids` in length and order. Null blobs are null;
@@ -3456,6 +3467,20 @@ impl BaseTable for NativeTable {
Ok(())
}
async fn list_bases(&self) -> Result<Vec<TableBase>> {
let dataset = self.dataset.get().await?;
let mut bases: Vec<&BasePath> = dataset.manifest().base_paths.values().collect();
bases.sort_by_key(|base| base.id);
Ok(bases
.into_iter()
.map(|base| TableBase {
path: base.path.clone(),
name: base.name.clone(),
is_dataset_root: base.is_dataset_root,
})
.collect())
}
async fn fetch_blobs(&self, column: &str, row_ids: &[u64]) -> Result<LargeBinaryArray> {
let dataset = self.dataset.get().await?;
crate::blob::take_blobs_aligned(&dataset, column, row_ids).await
+111 -27
View File
@@ -21,31 +21,27 @@ fn file_uri(path: &std::path::Path) -> String {
}
#[tokio::test]
async fn test_add_bases_accepts_named_and_dataset_root_entries() -> Result<()> {
async fn test_list_bases_reflects_added_bases() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let table = db.create_empty_table("t", empty_schema()).execute().await?;
let media = tmp.path().join("media");
let parent = tmp.path().join("parent");
std::fs::create_dir_all(&media).unwrap();
std::fs::create_dir_all(&parent).unwrap();
assert!(table.list_bases().await?.is_empty());
table
.add_bases([
TableBase {
path: file_uri(&media),
name: Some("media".into()),
is_dataset_root: false,
},
TableBase {
path: file_uri(&parent),
name: Some("parent".into()),
is_dataset_root: true,
},
])
.await
let media = tmp.path().join("media");
std::fs::create_dir_all(&media).unwrap();
let media_uri = file_uri(&media);
table.add_bases([&media_uri]).await?;
assert_eq!(
table.list_bases().await?,
vec![TableBase {
path: media_uri,
name: None,
is_dataset_root: false,
}]
);
Ok(())
}
#[tokio::test]
@@ -59,9 +55,26 @@ async fn test_add_bases_accepts_two_unnamed_paths() -> Result<()> {
let other = tmp.path().join("other");
std::fs::create_dir_all(&media).unwrap();
std::fs::create_dir_all(&other).unwrap();
table
.add_bases([&file_uri(&media), &file_uri(&other)])
.await
let media_uri = file_uri(&media);
let other_uri = file_uri(&other);
table.add_bases([&media_uri, &other_uri]).await?;
assert_eq!(
table.list_bases().await?,
vec![
TableBase {
path: media_uri,
name: None,
is_dataset_root: false,
},
TableBase {
path: other_uri,
name: None,
is_dataset_root: false,
},
]
);
Ok(())
}
#[tokio::test]
@@ -125,23 +138,94 @@ async fn test_add_bases_write_and_read_through_registered_base() -> Result<()> {
}
#[tokio::test]
async fn test_memory_add_bases_accepts_a_file_uri() -> Result<()> {
async fn test_add_bases_accepts_named_and_dataset_root_entries() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect("memory://").execute().await?;
let db = connect(tmp.path().join("db").to_str().unwrap())
.execute()
.await?;
let table = db.create_empty_table("t", empty_schema()).execute().await?;
let media = tmp.path().join("media");
let parent = tmp.path().join("parent");
std::fs::create_dir_all(&media).unwrap();
table.add_bases([file_uri(&media)]).await
std::fs::create_dir_all(&parent).unwrap();
let media_uri = file_uri(&media);
let parent_uri = file_uri(&parent);
table
.add_bases([
TableBase {
path: media_uri.clone(),
name: Some("media".into()),
is_dataset_root: false,
},
TableBase {
path: parent_uri.clone(),
name: Some("parent".into()),
is_dataset_root: true,
},
])
.await?;
assert_eq!(
table.list_bases().await?,
vec![
TableBase {
path: media_uri,
name: Some("media".into()),
is_dataset_root: false,
},
TableBase {
path: parent_uri,
name: Some("parent".into()),
is_dataset_root: true,
},
]
);
Ok(())
}
#[tokio::test]
async fn test_namespace_add_bases_accepts_a_file_uri() -> Result<()> {
async fn test_memory_list_bases_reflects_added_bases() -> Result<()> {
let tmp = tempdir().unwrap();
let db = connect("memory://").execute().await?;
let table = db.create_empty_table("t", empty_schema()).execute().await?;
assert!(table.list_bases().await?.is_empty());
let media = tmp.path().join("media");
std::fs::create_dir_all(&media).unwrap();
let media_uri = file_uri(&media);
table.add_bases([&media_uri]).await?;
assert_eq!(
table.list_bases().await?,
vec![TableBase {
path: media_uri,
name: None,
is_dataset_root: false,
}]
);
Ok(())
}
#[tokio::test]
async fn test_namespace_list_bases_reflects_added_bases() -> Result<()> {
let tmp = tempdir().unwrap();
let mut properties = std::collections::HashMap::new();
properties.insert("root".to_string(), tmp.path().to_str().unwrap().to_string());
let db = connect_namespace("dir", properties).execute().await?;
let table = db.create_empty_table("t", empty_schema()).execute().await?;
assert!(table.list_bases().await?.is_empty());
let media = tmp.path().join("media");
std::fs::create_dir_all(&media).unwrap();
table.add_bases([file_uri(&media)]).await
let media_uri = file_uri(&media);
table.add_bases([&media_uri]).await?;
assert_eq!(
table.list_bases().await?,
vec![TableBase {
path: media_uri,
name: None,
is_dataset_root: false,
}]
);
Ok(())
}