feat: builder API for list_tables, deprecate table_names

`Connection::list_tables` took a `lance_namespace::models::ListTablesRequest`
directly, so its generated shape -- including `identity`, `context` and
`include_declared`, none of which lancedb reads -- was part of the public API,
and Node had no binding at all.

Replaces it with a `ListTablesBuilder` carrying `page_token`, `limit` and
`namespace`, matching every other operation on `Connection`. This is a breaking
change for Rust callers. Node gains `listTables` with `ListTablesOptions` and
`ListTablesResponse`; Python's public API is unchanged, since it already had
`list_tables` everywhere.

`table_names` and `TableNamesBuilder` are deprecated. Its `start_after` takes a
table name rather than an opaque token, which cannot be pushed down into a store
that resumes from a continuation token.

Also fixes the page boundary in `ListingDatabase::list_tables`: the token was the
first name of the next page while resuming skips names at or before the token, so
one table was dropped per boundary. Walking `[a, b, c, d, e]` with a limit of 2
returned `[a, b, d, e]`.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Will Jones
2026-08-06 14:43:48 -07:00
parent 667cf32e78
commit 3c9c0becb6
15 changed files with 498 additions and 36 deletions
+74 -1
View File
@@ -506,6 +506,71 @@ Child namespace names and
***
### listTables()
#### listTables(options)
```ts
abstract listTables(options?): Promise<ListTablesResponse>
```
List a page of tables in this database.
Results may be paginated. To retrieve subsequent pages, pass the
`pageToken` returned by a previous call. A page may be shorter than
`limit` without being the last one, so walk until the response carries no
page token:
```ts
const names = [];
let pageToken = undefined;
do {
const page = await conn.listTables({ pageToken, limit: 100 });
names.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
```
##### Parameters
* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
Pagination options
(`pageToken`, `limit`).
##### Returns
`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
Table names and an optional token
for fetching the next page.
#### listTables(namespacePath, options)
```ts
abstract listTables(namespacePath?, options?): Promise<ListTablesResponse>
```
List a page of tables in this database.
##### Parameters
* **namespacePath?**: `string`[]
The namespace path to list tables from
(defaults to root namespace)
* **options?**: `Partial`&lt;[`ListTablesOptions`](../interfaces/ListTablesOptions.md)&gt;
Pagination options
(`pageToken`, `limit`).
##### Returns
`Promise`&lt;[`ListTablesResponse`](../interfaces/ListTablesResponse.md)&gt;
Table names and an optional token
for fetching the next page.
***
### openTable()
```ts
@@ -567,7 +632,7 @@ a "not supported" error.
***
### tableNames()
### ~~tableNames()~~
#### tableNames(options)
@@ -589,6 +654,10 @@ Tables will be returned in lexicographical order.
`Promise`&lt;`string`[]&gt;
##### Deprecated
Use [Connection.listTables](Connection.md#listtables) instead.
#### tableNames(namespacePath, options)
```ts
@@ -611,3 +680,7 @@ Tables will be returned in lexicographical order.
##### Returns
`Promise`&lt;`string`[]&gt;
##### Deprecated
Use [Connection.listTables](Connection.md#listtables) instead.
+2
View File
@@ -94,6 +94,8 @@
- [JobInfo](interfaces/JobInfo.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [ListTablesOptions](interfaces/ListTablesOptions.md)
- [ListTablesResponse](interfaces/ListTablesResponse.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MergeBlocker](interfaces/MergeBlocker.md)
- [MergeBranchResult](interfaces/MergeBranchResult.md)
@@ -0,0 +1,33 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListTablesOptions
# Interface: ListTablesOptions
## Properties
### limit?
```ts
optional limit: number;
```
An upper bound on how many tables to return.
A page may hold fewer than this and still not be the last one, so continue
while the response carries a page token rather than while pages are full.
***
### pageToken?
```ts
optional pageToken: string;
```
Token from a previous response for pagination.
The token is opaque: it carries whatever the database needs to resume, and
callers should not construct or interpret one.
@@ -0,0 +1,23 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ListTablesResponse
# Interface: ListTablesResponse
## Properties
### pageToken?
```ts
optional pageToken: string;
```
***
### tables
```ts
tables: string[];
```
+8 -3
View File
@@ -4,11 +4,16 @@
[@lancedb/lancedb](../globals.md) / TableNamesOptions
# Interface: TableNamesOptions
# Interface: ~~TableNamesOptions~~
## Deprecated
Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables)
instead.
## Properties
### limit?
### ~~limit?~~
```ts
optional limit: number;
@@ -18,7 +23,7 @@ An optional limit to the number of results to return.
***
### startAfter?
### ~~startAfter?~~
```ts
optional startAfter: string;
+57 -1
View File
@@ -4,7 +4,13 @@
import { readdirSync } from "fs";
import { Field, Float64, Schema } from "apache-arrow";
import * as tmp from "tmp";
import { Connection, Table, connect, connectNamespace } from "../lancedb";
import {
Connection,
ListTablesResponse,
Table,
connect,
connectNamespace,
} from "../lancedb";
import { LocalTable } from "../lancedb/table";
describe("when connecting", () => {
@@ -119,6 +125,56 @@ describe("given a connection", () => {
expect(tables).toEqual(["b", "c"]);
});
it("should list tables with a page token", async () => {
const db = await connect(tmpDir.name);
await db.createTable("b", [{ id: 1 }]);
await db.createTable("a", [{ id: 1 }]);
await db.createTable("c", [{ id: 1 }]);
const all = await db.listTables();
expect(all.tables).toEqual(["a", "b", "c"]);
expect(all.pageToken).toBeUndefined();
const first = await db.listTables({ limit: 1 });
expect(first.tables).toEqual(["a"]);
expect(first.pageToken).toBeDefined();
const second = await db.listTables({
limit: 1,
pageToken: first.pageToken,
});
expect(second.tables).toEqual(["b"]);
});
it("should visit every table exactly once when paging", async () => {
const db = await connect(tmpDir.name);
const created = ["a", "b", "c", "d", "e"];
for (const name of created) {
await db.createTable(name, [{ id: 1 }]);
}
const seen: string[] = [];
let pageToken: string | undefined = undefined;
do {
const page: ListTablesResponse = await db.listTables({
limit: 2,
pageToken,
});
seen.push(...page.tables);
pageToken = page.pageToken;
} while (pageToken);
expect(seen.sort()).toEqual(created);
});
it("should reject listTables on a closed connection", async () => {
const db = await connect(tmpDir.name);
db.close();
await expect(db.listTables()).rejects.toThrow("Connection is closed");
});
it("should create tables in v2 mode", async () => {
const db = await connect(tmpDir.name);
const data = [...Array(10000).keys()].map((i) => ({ id: i }));
+89
View File
@@ -25,12 +25,14 @@ import type {
JobDescription,
JobInfo,
ListNamespacesResponse,
ListTablesResponse,
} from "./native";
export type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
ListNamespacesResponse,
ListTablesResponse,
};
import { sanitizeTable } from "./sanitize";
import { LocalTable, Table } from "./table";
@@ -128,6 +130,10 @@ export interface OpenTableOptions {
indexCacheSize?: number;
}
/**
* @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
* instead.
*/
export interface TableNamesOptions {
/**
* If present, only return names that come lexicographically after the
@@ -141,6 +147,23 @@ export interface TableNamesOptions {
limit?: number;
}
export interface ListTablesOptions {
/**
* Token from a previous response for pagination.
*
* The token is opaque: it carries whatever the database needs to resume, and
* callers should not construct or interpret one.
*/
pageToken?: string;
/**
* An upper bound on how many tables to return.
*
* A page may hold fewer than this and still not be the last one, so continue
* while the response carries a page token rather than while pages are full.
*/
limit?: number;
}
export interface ListNamespacesOptions {
/** Token from a previous response for pagination. */
pageToken?: string;
@@ -225,6 +248,7 @@ export abstract class Connection {
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point (backwards compatibility)
*
* @deprecated Use {@link Connection.listTables} instead.
*/
abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
/**
@@ -235,12 +259,54 @@ export abstract class Connection {
* @param {Partial<TableNamesOptions>} options - options to control the
* paging / start point
*
* @deprecated Use {@link Connection.listTables} instead.
*/
abstract tableNames(
namespacePath?: string[],
options?: Partial<TableNamesOptions>,
): Promise<string[]>;
/**
* List a page of tables in this database.
*
* Results may be paginated. To retrieve subsequent pages, pass the
* `pageToken` returned by a previous call. A page may be shorter than
* `limit` without being the last one, so walk until the response carries no
* page token:
*
* ```ts
* const names = [];
* let pageToken = undefined;
* do {
* const page = await conn.listTables({ pageToken, limit: 100 });
* names.push(...page.tables);
* pageToken = page.pageToken;
* } while (pageToken);
* ```
*
* @param {Partial<ListTablesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListTablesResponse>} Table names and an optional token
* for fetching the next page.
*/
abstract listTables(
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse>;
/**
* List a page of tables in this database.
*
* @param {string[]} namespacePath - The namespace path to list tables from
* (defaults to root namespace)
* @param {Partial<ListTablesOptions>} options - Pagination options
* (`pageToken`, `limit`).
* @returns {Promise<ListTablesResponse>} Table names and an optional token
* for fetching the next page.
*/
abstract listTables(
namespacePath?: string[],
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse>;
/**
* Open a table in the database.
* @param {string} name - The name of the table
@@ -523,6 +589,29 @@ export class LocalConnection extends Connection {
);
}
async listTables(
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
options?: Partial<ListTablesOptions>,
): Promise<ListTablesResponse> {
// Detect if first argument is namespacePath array or options object
let namespacePath: string[] | undefined;
let listTablesOptions: Partial<ListTablesOptions> | undefined;
if (Array.isArray(namespacePathOrOptions)) {
namespacePath = namespacePathOrOptions;
listTablesOptions = options;
} else {
namespacePath = undefined;
listTablesOptions = namespacePathOrOptions;
}
return this.inner.listTables(
namespacePath ?? [],
listTablesOptions?.pageToken,
listTablesOptions?.limit,
);
}
async openTable(
name: string,
namespacePath?: string[],
+2
View File
@@ -74,11 +74,13 @@ export {
Connection,
CreateTableOptions,
TableNamesOptions,
ListTablesOptions,
OpenTableOptions,
ListNamespacesOptions,
CreateNamespaceOptions,
DropNamespaceOptions,
ListNamespacesResponse,
ListTablesResponse,
CreateNamespaceResponse,
DropNamespaceResponse,
DescribeNamespaceResponse,
+31
View File
@@ -36,6 +36,12 @@ pub struct ListNamespacesResponse {
pub page_token: Option<String>,
}
#[napi(object)]
pub struct ListTablesResponse {
pub tables: Vec<String>,
pub page_token: Option<String>,
}
#[napi(object)]
pub struct CreateNamespaceResponse {
pub properties: Option<HashMap<String, String>>,
@@ -189,6 +195,8 @@ impl Connection {
/// List all tables in the dataset.
#[napi(catch_unwind)]
// Deprecated in favour of `list_tables`, but still exposed to JavaScript.
#[allow(deprecated)]
pub async fn table_names(
&self,
namespace_path: Option<Vec<String>>,
@@ -206,6 +214,29 @@ impl Connection {
op.execute().await.default_error()
}
/// List a page of tables in the database.
#[napi(catch_unwind)]
pub async fn list_tables(
&self,
namespace_path: Option<Vec<String>>,
page_token: Option<String>,
limit: Option<u32>,
) -> napi::Result<ListTablesResponse> {
let mut op = self.get_inner()?.list_tables();
op = op.namespace(namespace_path.unwrap_or_default());
if let Some(page_token) = page_token {
op = op.page_token(page_token);
}
if let Some(limit) = limit {
op = op.limit(limit);
}
let resp = op.execute().await.default_error()?;
Ok(ListTablesResponse {
tables: resp.tables,
page_token: resp.page_token,
})
}
/// Create table from a Apache Arrow IPC (file) buffer.
///
/// Parameters:
+13 -8
View File
@@ -121,6 +121,8 @@ impl Connection {
}
#[pyo3(signature = (namespace_path=None, start_after=None, limit=None))]
// Deprecated in favour of `list_tables`, but still exposed to Python.
#[allow(deprecated)]
pub fn table_names(
self_: PyRef<'_, Self>,
namespace_path: Option<Vec<String>>,
@@ -505,14 +507,17 @@ impl Connection {
let inner = self_.get_inner()?.clone();
let py = self_.py();
future_into_py(py, async move {
use lance_namespace::models::ListTablesRequest;
let request = ListTablesRequest {
id: namespace_path,
page_token,
limit: limit.map(|l| l as i32),
..Default::default()
};
let response = inner.list_tables(request).await.infer_error()?;
let mut request = inner.list_tables();
if let Some(namespace_path) = namespace_path {
request = request.namespace(namespace_path);
}
if let Some(page_token) = page_token {
request = request.page_token(page_token);
}
if let Some(limit) = limit {
request = request.limit(limit);
}
let response = request.execute().await.infer_error()?;
Python::attach(|py| -> PyResult<Py<PyDict>> {
let dict = PyDict::new(py);
dict.set_item("tables", response.tables)?;
+1 -1
View File
@@ -27,7 +27,7 @@ async fn main() -> Result<()> {
// --8<-- [end:connect]
// --8<-- [start:list_names]
println!("{:?}", db.table_names().execute().await?);
println!("{:?}", db.list_tables().execute().await?.tables);
// --8<-- [end:list_names]
let tbl = create_table(&db).await?;
create_index(&tbl).await?;
+146 -4
View File
@@ -73,11 +73,13 @@ fn set_storage_options_provider(
}
/// A builder for configuring a [`Connection::table_names`] operation
#[deprecated(note = "Use Connection::list_tables instead")]
pub struct TableNamesBuilder {
parent: Arc<dyn Database>,
request: TableNamesRequest,
}
#[allow(deprecated)]
impl TableNamesBuilder {
fn new(parent: Arc<dyn Database>) -> Self {
Self {
@@ -115,6 +117,57 @@ impl TableNamesBuilder {
}
}
/// A builder for configuring a [`Connection::list_tables`] operation
pub struct ListTablesBuilder {
parent: Arc<dyn Database>,
request: ListTablesRequest,
}
impl ListTablesBuilder {
fn new(parent: Arc<dyn Database>) -> Self {
Self {
parent,
request: ListTablesRequest {
// The root namespace is an empty path, not an absent one: a
// namespace-backed database rejects a request that names no namespace.
id: Some(Vec::new()),
..Default::default()
},
}
}
/// Resume listing from a previous page.
///
/// Pass the `page_token` from the previous [`ListTablesResponse`]. The token is
/// opaque: it carries whatever the database needs to resume, and callers should
/// not construct or interpret one. A response whose token is `None` or empty is
/// the end of the listing.
pub fn page_token(mut self, page_token: impl Into<String>) -> Self {
self.request.page_token = Some(page_token.into());
self
}
/// An upper bound on how many tables to return.
///
/// A page may hold fewer than this and still not be the last one, so continue
/// while the response carries a page token rather than while pages are full.
pub fn limit(mut self, limit: u32) -> Self {
self.request.limit = Some(i32::try_from(limit).unwrap_or(i32::MAX));
self
}
/// Set the namespace path to list tables from. Defaults to the root namespace.
pub fn namespace(mut self, namespace_path: Vec<String>) -> Self {
self.request.id = Some(namespace_path);
self
}
/// Execute the list tables operation
pub async fn execute(self) -> Result<ListTablesResponse> {
self.parent.clone().list_tables(self.request).await
}
}
#[derive(Clone, Debug)]
pub struct OpenTableBuilder {
parent: Arc<dyn Database>,
@@ -409,7 +462,9 @@ impl Connection {
///
/// The names will be returned in lexicographical order (ascending)
///
/// The parameters `page_token` and `limit` can be used to paginate the results
/// The parameters `start_after` and `limit` can be used to paginate the results
#[deprecated(note = "Use Connection::list_tables instead")]
#[allow(deprecated)]
pub fn table_names(&self) -> TableNamesBuilder {
TableNamesBuilder::new(self.internal.clone())
}
@@ -633,9 +688,32 @@ impl Connection {
self.internal.namespace_client_config().await
}
/// List tables with pagination support
pub async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
self.internal.list_tables(request).await
/// List the tables in the database, a page at a time
///
/// ```
/// # use lancedb::Connection;
/// # async fn list_all(conn: &Connection) -> Result<Vec<String>, lancedb::Error> {
/// let mut names = Vec::new();
/// let mut token = None;
/// loop {
/// let mut request = conn.list_tables().limit(100);
/// if let Some(token) = token {
/// request = request.page_token(token);
/// }
/// let page = request.execute().await?;
/// names.extend(page.tables);
/// // A page may be short without being the last one, so the token is what ends
/// // the walk.
/// token = page.page_token.filter(|token| !token.is_empty());
/// if token.is_none() {
/// break;
/// }
/// }
/// # Ok(names)
/// # }
/// ```
pub fn list_tables(&self) -> ListTablesBuilder {
ListTablesBuilder::new(self.internal.clone())
}
/// Get the in-memory embedding registry.
@@ -1291,6 +1369,8 @@ mod test_utils {
}
#[cfg(test)]
// `table_names` is deprecated but still supported, so its tests still call it.
#[allow(deprecated)]
mod tests {
use arrow_schema::{DataType, Field, Schema};
use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
@@ -1633,6 +1713,68 @@ mod tests {
assert_eq!(tables, names[..7]);
}
#[tokio::test]
async fn test_list_tables_paginates() {
let tc = new_test_connection().await.unwrap();
let db = tc.connection;
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
let mut names = Vec::with_capacity(25);
for _ in 0..25 {
let name = uuid::Uuid::new_v4().to_string();
names.push(name.clone());
db.create_empty_table(name, schema.clone())
.execute()
.await
.unwrap();
}
names.sort();
let page = db.list_tables().limit(10).execute().await.unwrap();
assert_eq!(page.tables, names[..10]);
assert_eq!(page.page_token.as_deref(), Some(names[9].as_str()));
// Walking in pages has to reach every table exactly once, with nothing lost
// at a page boundary.
let mut seen = Vec::with_capacity(names.len());
let mut page_token = None;
loop {
let mut request = db.list_tables().limit(10);
if let Some(token) = page_token {
request = request.page_token(token);
}
let page = request.execute().await.unwrap();
seen.extend(page.tables);
page_token = page.page_token.filter(|token| !token.is_empty());
if page_token.is_none() {
break;
}
}
assert_eq!(seen, names);
}
#[tokio::test]
async fn test_list_tables_exhausted_has_no_token() {
let tc = new_test_connection().await.unwrap();
let db = tc.connection;
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
for i in 0..3 {
db.create_empty_table(format!("table{i}"), schema.clone())
.execute()
.await
.unwrap();
}
// A limit the listing does not fill leaves no token behind.
let page = db.list_tables().limit(10).execute().await.unwrap();
assert_eq!(page.tables.len(), 3);
assert_eq!(page.page_token, None);
// Neither does one that exactly exhausts it.
let page = db.list_tables().limit(3).execute().await.unwrap();
assert_eq!(page.tables.len(), 3);
assert_eq!(page.page_token, None);
}
#[tokio::test]
async fn test_open_table() {
let tc = new_test_connection().await.unwrap();
+7 -9
View File
@@ -1018,17 +1018,15 @@ impl Database for ListingDatabase {
f.drain(0..index);
}
// Determine if there's a next page
let next_page_token = if let Some(limit) = request.limit {
if f.len() > limit as usize {
let token = f[limit as usize].clone();
// Determine if there's a next page. The token is the last name of this page,
// not the first of the next one: the next page resumes strictly after the
// token, so naming the next page's first entry would skip it.
let next_page_token = match request.limit {
Some(limit) if f.len() > limit as usize => {
f.truncate(limit as usize);
Some(token)
} else {
None
f.last().cloned()
}
} else {
None
_ => None,
};
Ok(ListTablesResponse {
+4 -1
View File
@@ -621,7 +621,10 @@ impl Database for LanceNamespaceDatabase {
}
#[cfg(test)]
#[cfg(not(windows))] // TODO: support windows for lance-namespace
#[cfg(not(windows))]
// TODO: support windows for lance-namespace
// `table_names` is deprecated but still supported, so its tests still call it.
#[allow(deprecated)]
mod tests {
use super::*;
use crate::connect_namespace;
+8 -8
View File
@@ -1100,6 +1100,8 @@ impl From<StorageOptions> for RemoteOptions {
}
#[cfg(test)]
// `table_names` is deprecated but still supported, so its tests still call it.
#[allow(deprecated)]
mod tests {
use super::{NamespaceHeaderProviderContext, build_cache_key};
use std::collections::HashMap;
@@ -2177,10 +2179,9 @@ mod tests {
// List tables in the child namespace
let list_response = conn
.list_tables(ListTablesRequest {
id: Some(namespace.clone()),
..Default::default()
})
.list_tables()
.namespace(namespace.clone())
.execute()
.await
.expect("Failed to list tables");
assert_eq!(list_response.tables, vec!["test_table"]);
@@ -2251,10 +2252,9 @@ mod tests {
// List tables in the child namespace
let list_response = conn
.list_tables(ListTablesRequest {
id: Some(namespace.clone()),
..Default::default()
})
.list_tables()
.namespace(namespace.clone())
.execute()
.await
.unwrap();
assert_eq!(list_response.tables.len(), 3);