feat!: return lazy iterators for table and namespace listings (#4599)

Make table and namespace listings consistent with other resource
listings across Rust, Python, and TypeScript. Callers can iterate over
individual names across all pages, with configurable page sizes and
continuation tokens.

In order to avoid breaking existing Python users, this change adds some
compatibility methods to the returned Python iterators. Code such as:
```
for t in db.list_tables().tables:
    print(t)
```
will continue to work as expected because the iterator has a `tables`
field and a `page_token` field, which function as expected to reveal the
first page of results. In other words, it is never necessary to use the
iterator __as an iterator__. It may be used as if it were an old-style
response object. (As the iterator progresses, the `tables` list may
switch to the second, third, etc. page of results.) Similar
compatibility code allows the async python API to keep working as well.

nodejs/__test__/connection.test.ts

@@ -5,15 +5,15 @@ import { mkdirSync, readdirSync } from "fs";
 import { join } from "path";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import {
-  Connection,
-  ListTablesResponse,
-  Table,
-  connect,
-  connectNamespace,
-} from "../lancedb";
+import { Connection, Table, connect, connectNamespace } from "../lancedb";
 import { LocalTable } from "../lancedb/table";
 
+async function collectNames(items: AsyncIterable<string>): Promise<string[]> {
+  const names: string[] = [];
+  for await (const name of items) names.push(name);
+  return names;
+}
+
 describe("when connecting", () => {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -135,7 +135,7 @@ describe("given a connection", () => {
     await db.close();
     expect(db.isOpen()).toBe(false);
     await expect(db.tableNames()).rejects.toThrow("Connection is closed");
-    await expect(db.listTables()).rejects.toThrow("Connection is closed");
+    expect(() => db.listTables()).toThrow("Connection is closed");
     await expect(db.renameTable("a", "b")).rejects.toThrow(
       "Connection is closed",
     );
@@ -247,15 +247,15 @@ describe("given a connection", () => {
         await db.createTable(name, [{ id: 1 }]);
       }
 
-      const first = await db.listTables(namespacePath, { limit: 1 });
-      expect(first.tables).toEqual(["a"]);
-      expect(first.pageToken).toBeDefined();
+      const first = db.listTables(namespacePath, { pageLimit: 1 });
+      expect(await first.next()).toEqual({ done: false, value: "a" });
+      expect(first.pageToken()).toBeDefined();
 
-      const second = await db.listTables(namespacePath, {
-        limit: 1,
-        pageToken: first.pageToken,
+      const second = db.listTables(namespacePath, {
+        pageLimit: 1,
+        pageToken: first.pageToken(),
       });
-      expect(second.tables).toEqual(["b"]);
+      expect(await second.next()).toEqual({ done: false, value: "b" });
     });
   });
   it("should keep the existing table when mode is exist_ok", async () => {
@@ -299,19 +299,18 @@ describe("given a connection", () => {
     await db.createTable("a", [{ id: 1 }]);
     await db.createTable("c", [{ id: 1 }]);
 
-    const all = await db.listTables();
-    expect(all.tables).toEqual(["a", "b", "c"]);
-    expect(all.pageToken).toBeUndefined();
+    expect(await collectNames(db.listTables())).toEqual(["a", "b", "c"]);
 
-    const first = await db.listTables({ limit: 1 });
-    expect(first.tables).toEqual(["a"]);
-    expect(first.pageToken).toBeDefined();
+    const first = db.listTables([], { pageLimit: 1 });
+    expect(first.numPageResults()).toBe(0);
+    expect(await first.next()).toEqual({ done: false, value: "a" });
+    expect(first.pageToken()).toBeDefined();
 
-    const second = await db.listTables({
-      limit: 1,
-      pageToken: first.pageToken,
+    const second = db.listTables([], {
+      pageLimit: 1,
+      pageToken: first.pageToken(),
     });
-    expect(second.tables).toEqual(["b"]);
+    expect(await second.next()).toEqual({ done: false, value: "b" });
   });
 
   it("should visit every table exactly once when walking pages", async () => {
@@ -322,18 +321,9 @@ describe("given a connection", () => {
       await db.createTable(name, [{ id: 1 }]);
     }
 
-    const seen: string[] = [];
-    let pageToken: string | undefined = undefined;
-    do {
-      const page: ListTablesResponse = await db.listTables({
-        limit: 2,
-        pageToken,
-      });
-      seen.push(...page.tables);
-      pageToken = page.pageToken;
-    } while (pageToken);
-
-    expect(seen).toEqual(created);
+    expect(await collectNames(db.listTables([], { pageLimit: 2 }))).toEqual(
+      created,
+    );
   });
 
   it.each([2 ** 32, 2 ** 32 + 1, Number.MAX_SAFE_INTEGER])(
@@ -347,11 +337,10 @@ describe("given a connection", () => {
       ]) {
         await expect(names).resolves.toEqual(["a", "b"]);
       }
-      for (const page of [
-        db.listTables({ limit }),
-        db.listTables([], { limit }),
-      ]) {
-        await expect(page).resolves.toEqual({ tables: ["a", "b"] });
+      for (const namespacePath of [undefined, []]) {
+        expect(() =>
+          db.listTables(namespacePath, { pageLimit: limit }),
+        ).toThrow(/limit/);
       }
     },
   );
@@ -366,8 +355,11 @@ describe("given a connection", () => {
   ])("should reject an invalid table listing limit %s", async (limit) => {
     await expect(db.tableNames({ limit })).rejects.toThrow(/limit/);
     await expect(db.tableNames([], { limit })).rejects.toThrow(/limit/);
-    await expect(db.listTables({ limit })).rejects.toThrow(/limit/);
-    await expect(db.listTables([], { limit })).rejects.toThrow(/limit/);
+    for (const namespacePath of [undefined, []]) {
+      expect(() => db.listTables(namespacePath, { pageLimit: limit })).toThrow(
+        /limit/,
+      );
+    }
   });
 
   it("should preserve zero, small and unsigned-32-bit table listing limits", async () => {
@@ -379,11 +371,21 @@ describe("given a connection", () => {
       [2 ** 32 - 1, ["a", "b"]],
     ] as const) {
       await expect(db.tableNames({ limit })).resolves.toEqual(names);
-      const page = await db.listTables({ limit });
-      expect(page.tables).toEqual(names);
     }
     await expect(db.tableNames()).resolves.toEqual(["a", "b"]);
-    await expect(db.listTables()).resolves.toEqual({ tables: ["a", "b"] });
+    for (const namespacePath of [undefined, []]) {
+      for (const pageLimit of [1, 2 ** 31 - 1]) {
+        expect(
+          await collectNames(db.listTables(namespacePath, { pageLimit })),
+        ).toEqual(["a", "b"]);
+      }
+      for (const pageLimit of [0, 2 ** 31, 2 ** 32 - 1]) {
+        expect(() => db.listTables(namespacePath, { pageLimit })).toThrow(
+          /limit/,
+        );
+      }
+    }
+    expect(await collectNames(db.listTables())).toEqual(["a", "b"]);
   });
 
   it("should list all names and use a default page size of 100", async () => {
@@ -396,20 +398,17 @@ describe("given a connection", () => {
     }
     await expect(db.tableNames()).resolves.toEqual(names);
     await expect(db.tableNames({ limit: 0 })).resolves.toEqual([]);
-    const first = await db.listTables();
-    expect(first.tables).toEqual(names.slice(0, 100));
-    expect(first.pageToken).toBeDefined();
-    const second = await db.listTables({ pageToken: first.pageToken });
-    expect(second.tables).toEqual(names.slice(100));
-    expect(second.pageToken).toBeUndefined();
-    await expect(db.listTables({ limit: 200 })).resolves.toEqual({
-      tables: names,
-      pageToken: undefined,
-    });
-    await expect(db.listTables({ limit: 0 })).resolves.toEqual({
-      tables: [],
-      pageToken: undefined,
-    });
+    const first = db.listTables();
+    const firstPage: string[] = [];
+    for (let i = 0; i < 100; i++) firstPage.push((await first.next()).value!);
+    expect(firstPage).toEqual(names.slice(0, 100));
+    expect(first.pageToken()).toBeDefined();
+    expect(first.numPageResults()).toBe(0);
+    expect(await collectNames(first)).toEqual(names.slice(100));
+    expect(await collectNames(db.listTables([], { pageLimit: 200 }))).toEqual(
+      names,
+    );
+    expect(() => db.listTables([], { pageLimit: 0 })).toThrow(/limit/);
   });
 
   it("should list tables in a namespace", async () => {
@@ -420,12 +419,8 @@ describe("given a connection", () => {
     await db.createNamespace(["child"]);
     await db.createTable("nested", [{ id: 1 }], ["child"]);
 
-    await expect(db.listTables(["child"])).resolves.toEqual(
-      expect.objectContaining({ tables: ["nested"] }),
-    );
-    await expect(db.listTables()).resolves.toEqual(
-      expect.objectContaining({ tables: [] }),
-    );
+    expect(await collectNames(db.listTables(["child"]))).toEqual(["nested"]);
+    expect(await collectNames(db.listTables())).toEqual([]);
   });
 
   it("should create tables in v2 mode", async () => {
@@ -719,22 +714,35 @@ describe("namespaces", () => {
   it("should list namespaces created at the root", async () => {
     await db.createNamespace(["alpha"]);
     await db.createNamespace(["beta"]);
-    const list = await db.listNamespaces();
-    expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
+    const list = await collectNames(db.listNamespaces());
+    expect(list).toEqual(expect.arrayContaining(["alpha", "beta"]));
   });
 
   it("should list child namespaces under a parent", async () => {
     await db.createNamespace(["parent"]);
     await db.createNamespace(["parent", "child"]);
-    const list = await db.listNamespaces(["parent"]);
-    expect(list.namespaces).toContain("child");
+    const list = await collectNames(db.listNamespaces(["parent"]));
+    expect(list).toContain("child");
+  });
+
+  it("should resume a namespace listing from its page token", async () => {
+    await db.createNamespace(["alpha"]);
+    await db.createNamespace(["beta"]);
+    const first = db.listNamespaces([], { pageLimit: 1 });
+    expect(first.numPageResults()).toBe(0);
+    expect((await first.next()).value).toBe("alpha");
+    const second = db.listNamespaces([], {
+      pageLimit: 1,
+      pageToken: first.pageToken(),
+    });
+    expect(await collectNames(second)).toEqual(["beta"]);
   });
 
   it("should drop a namespace", async () => {
     await db.createNamespace(["ephemeral"]);
     await db.dropNamespace(["ephemeral"]);
-    const list = await db.listNamespaces();
-    expect(list.namespaces).not.toContain("ephemeral");
+    const list = await collectNames(db.listNamespaces());
+    expect(list).not.toContain("ephemeral");
   });
 
   it("should raise an error on any namespace op after close", async () => {
@@ -742,7 +750,7 @@ describe("namespaces", () => {
     await expect(db.describeNamespace(["foo"])).rejects.toThrow(
       "Connection is closed",
     );
-    await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
+    expect(() => db.listNamespaces()).toThrow("Connection is closed");
     await expect(db.createNamespace(["foo"])).rejects.toThrow(
       "Connection is closed",
     );

nodejs/lancedb/connection.ts

@@ -30,16 +30,12 @@ import type {
   DescribeNamespaceResponse,
   DropNamespaceResponse,
   JobInfo,
-  ListNamespacesResponse,
-  ListTablesResponse,
 } from "./native";
 import { ViewDescription, viewDescriptionFromNative } from "./view";
 export type {
   CreateNamespaceResponse,
   DescribeNamespaceResponse,
   DropNamespaceResponse,
-  ListNamespacesResponse,
-  ListTablesResponse,
 };
 import { sanitizeTable } from "./sanitize";
 import { LocalTable, Table } from "./table";
@@ -141,7 +137,7 @@ export interface OpenTableOptions {
 }
 
 /**
- * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
+ * @deprecated Use {@link ListingOptions} with {@link Connection.listTables}
  * instead.
  */
 export interface TableNamesOptions {
@@ -157,32 +153,6 @@ export interface TableNamesOptions {
   limit?: number;
 }
 
-export interface ListTablesOptions {
-  /**
-   * Token from a previous response, to resume listing where it left off.
-   *
-   * The token is opaque: it carries whatever the database needs to resume, and
-   * callers should not construct or interpret one.
-   */
-  pageToken?: string;
-  /**
-   * An upper bound on how many tables to return.
-   * Defaults to 100. Zero returns an empty page without a continuation token.
-   *
-   * A page may hold fewer than this and still not be the last one, so keep
-   * going while the response carries a page token rather than while pages are
-   * full.
-   */
-  limit?: number;
-}
-
-export interface ListNamespacesOptions {
-  /** Token from a previous response for pagination. */
-  pageToken?: string;
-  /** An optional limit to the number of results to return. */
-  limit?: number;
-}
-
 export interface CreateNamespaceOptions {
   /** Creation mode. */
   mode?: "create" | "exist_ok" | "overwrite";
@@ -278,45 +248,18 @@ export abstract class Connection {
     options?: Partial<TableNamesOptions>,
   ): Promise<string[]>;
 
-  /**
-   * List a page of the tables in this database.
-   *
-   * To retrieve the tables after the page, pass the `pageToken` the response
-   * carries back in. A page can be shorter than `limit` without being the last
-   * one, so walk until a response carries no page token:
-   *
+  /** Lazily list tables in a namespace, requesting 100 per page by default.
+   * @example
    * ```ts
-   * const names = [];
-   * let pageToken = undefined;
-   * do {
-   *   const page = await conn.listTables({ pageToken, limit: 100 });
-   *   names.push(...page.tables);
-   *   pageToken = page.pageToken;
-   * } while (pageToken);
+   * for await (const name of conn.listTables([], { pageLimit: 20 })) {
+   *   console.log(name);
+   * }
    * ```
-   *
-   * @param {Partial<ListTablesOptions>} options - Pagination options
-   *   (`pageToken`, `limit`).
-   * @returns {Promise<ListTablesResponse>} A page of table names and an
-   *   optional token for the tables after it.
-   */
-  abstract listTables(
-    options?: Partial<ListTablesOptions>,
-  ): Promise<ListTablesResponse>;
-  /**
-   * List a page of the tables in this database.
-   *
-   * @param {string[]} namespacePath - The namespace path to list tables from
-   *   (defaults to root namespace)
-   * @param {Partial<ListTablesOptions>} options - Pagination options
-   *   (`pageToken`, `limit`).
-   * @returns {Promise<ListTablesResponse>} A page of table names and an
-   *   optional token for the tables after it.
    */
   abstract listTables(
     namespacePath?: string[],
-    options?: Partial<ListTablesOptions>,
-  ): Promise<ListTablesResponse>;
+    options?: ListingOptions,
+  ): Listing<string>;
 
   /**
    * Open a table in the database.
@@ -542,23 +485,11 @@ export abstract class Connection {
     namespacePath: string[],
   ): Promise<DescribeNamespaceResponse>;
 
-  /**
-   * List the immediate child namespaces under the given parent.
-   *
-   * Results may be paginated. To retrieve subsequent pages, pass the
-   * `pageToken` returned by a previous call.
-   *
-   * @param {string[]} namespacePath - The parent namespace path. Defaults
-   *   to the root namespace if omitted.
-   * @param {Partial<ListNamespacesOptions>} options - Pagination options
-   *   (`pageToken`, `limit`).
-   * @returns {Promise<ListNamespacesResponse>} Child namespace names and
-   *   an optional token for fetching the next page.
-   */
+  /** Lazily list immediate child namespaces, 100 per page by default. */
   abstract listNamespaces(
     namespacePath?: string[],
-    options?: Partial<ListNamespacesOptions>,
-  ): Promise<ListNamespacesResponse>;
+    options?: ListingOptions,
+  ): Listing<string>;
 
   /**
    * Create a new namespace at the given path.
@@ -821,22 +752,17 @@ export class LocalConnection extends Connection {
     );
   }
 
-  async listTables(
-    namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
-    options?: Partial<ListTablesOptions>,
-  ): Promise<ListTablesResponse> {
-    // Detect if first argument is namespacePath array or options object
-    const namespacePath = Array.isArray(namespacePathOrOptions)
-      ? namespacePathOrOptions
-      : undefined;
-    const listTablesOptions = Array.isArray(namespacePathOrOptions)
-      ? options
-      : (namespacePathOrOptions ?? options);
-
-    return this.inner.listTables(
-      namespacePath ?? [],
-      listTablesOptions?.pageToken,
-      normalizeTableListingLimit(listTablesOptions?.limit),
+  listTables(
+    namespacePath?: string[],
+    options: ListingOptions = {},
+  ): Listing<string> {
+    validateListingOptions(options);
+    return new Listing(
+      this.inner.listTables(
+        namespacePath ?? [],
+        options.pageToken,
+        options.pageLimit,
+      ),
     );
   }
 
@@ -1039,12 +965,15 @@ export class LocalConnection extends Connection {
 
   listNamespaces(
     namespacePath?: string[],
-    options?: Partial<ListNamespacesOptions>,
-  ): Promise<ListNamespacesResponse> {
-    return this.inner.listNamespaces(
-      namespacePath ?? [],
-      options?.pageToken,
-      options?.limit,
+    options: ListingOptions = {},
+  ): Listing<string> {
+    validateListingOptions(options);
+    return new Listing(
+      this.inner.listNamespaces(
+        namespacePath ?? [],
+        options.pageToken,
+        options.pageLimit,
+      ),
     );
   }
 

nodejs/lancedb/index.ts

@@ -97,13 +97,9 @@ export {
   Connection,
   CreateTableOptions,
   TableNamesOptions,
-  ListTablesOptions,
   OpenTableOptions,
-  ListNamespacesOptions,
   CreateNamespaceOptions,
   DropNamespaceOptions,
-  ListNamespacesResponse,
-  ListTablesResponse,
   CreateNamespaceResponse,
   DropNamespaceResponse,
   DescribeNamespaceResponse,

nodejs/src/connection.rs

@@ -16,8 +16,7 @@ use crate::table::Table;
 use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, connect_namespace};
 
 use lance_namespace::models::{
-    CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
-    ListTablesRequest,
+    CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest,
 };
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
@@ -31,18 +30,6 @@ pub struct DescribeNamespaceResponse {
     pub properties: Option<HashMap<String, String>>,
 }
 
-#[napi(object)]
-pub struct ListNamespacesResponse {
-    pub namespaces: Vec<String>,
-    pub page_token: Option<String>,
-}
-
-#[napi(object)]
-pub struct ListTablesResponse {
-    pub tables: Vec<String>,
-    pub page_token: Option<String>,
-}
-
 #[napi(object)]
 pub struct CreateNamespaceResponse {
     pub properties: Option<HashMap<String, String>>,
@@ -239,31 +226,20 @@ impl Connection {
         op.execute().await.default_error()
     }
 
-    /// List a page of tables in the database.
+    /// Lazily list tables in the database.
     #[napi(catch_unwind)]
-    pub async fn list_tables(
+    pub fn list_tables(
         &self,
         namespace_path: Option<Vec<String>>,
         page_token: Option<String>,
         limit: Option<u32>,
-    ) -> napi::Result<ListTablesResponse> {
-        let request = ListTablesRequest {
-            // The root namespace is an empty path, not an absent one: a namespace-backed
-            // database rejects a request that names no namespace.
-            id: Some(namespace_path.unwrap_or_default()),
-            page_token,
-            limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
-            ..Default::default()
-        };
-        let response = self
-            .get_inner()?
-            .list_tables(request)
-            .await
-            .default_error()?;
-        Ok(ListTablesResponse {
-            tables: response.tables,
-            page_token: response.page_token,
-        })
+    ) -> napi::Result<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?.list_tables(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, limit),
+            ),
+        ))
     }
 
     /// Create table from a Apache Arrow IPC (file) buffer.
@@ -663,28 +639,19 @@ impl Connection {
     }
 
     #[napi(catch_unwind)]
-    /// List child namespaces under the given namespace path
-    pub async fn list_namespaces(
+    /// Lazily list child namespaces under the given namespace path
+    pub fn list_namespaces(
         &self,
         namespace_path: Option<Vec<String>>,
         page_token: Option<String>,
         limit: Option<u32>,
-    ) -> napi::Result<ListNamespacesResponse> {
-        let req = ListNamespacesRequest {
-            id: namespace_path,
-            page_token,
-            limit: checked_limit(limit)?,
-            ..Default::default()
-        };
-        let resp = self
-            .get_inner()?
-            .list_namespaces(req)
-            .await
-            .default_error()?;
-        Ok(ListNamespacesResponse {
-            namespaces: resp.namespaces,
-            page_token: resp.page_token,
-        })
+    ) -> napi::Result<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?.list_namespaces(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, limit),
+            ),
+        ))
     }
 
     #[napi(catch_unwind)]
@@ -787,33 +754,3 @@ impl Connection {
             .default_error()
     }
 }
-
-fn checked_limit(limit: Option<u32>) -> napi::Result<Option<i32>> {
-    limit
-        .map(i32::try_from)
-        .transpose()
-        .map_err(|_| napi::Error::from_reason("limit must not exceed 2147483647"))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::checked_limit;
-
-    #[test]
-    fn checked_limit_rejects_values_above_i32_max() {
-        // u32 values above i32::MAX wrapped negative through `as i32`,
-        // sending a negative page size to the server.
-        assert!(checked_limit(Some(3_000_000_000)).is_err());
-        assert!(checked_limit(Some(u32::MAX)).is_err());
-    }
-
-    #[test]
-    fn checked_limit_passes_boundary_and_missing_values() {
-        assert_eq!(
-            checked_limit(Some(i32::MAX as u32)).unwrap(),
-            Some(i32::MAX)
-        );
-        assert_eq!(checked_limit(Some(0)).unwrap(), Some(0));
-        assert_eq!(checked_limit(None).unwrap(), None);
-    }
-}