refactor!: implement lazy iterators for several APIs (#4459)

Implement lazy iterators for:
- secrets
- views
- jobs
- functions
- materialized views

python/python/lancedb/_lancedb.pyi

@@ -155,9 +155,13 @@ class Connection(object):
     async def get_function(
         self, name: str, version: str, namespace_path: Optional[List[str]] = None
     ) -> str: ...
-    async def list_functions(
-        self, namespace_path: Optional[List[str]] = None
-    ) -> List[str]: ...
+    def list_functions(
+        self,
+        namespace_path: Optional[List[str]] = None,
+        *,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> FunctionListing: ...
     async def drop_function(
         self, name: str, version: str, namespace_path: Optional[List[str]] = None
     ) -> bool: ...
@@ -170,9 +174,13 @@ class Connection(object):
     async def alter_secret(
         self, name: str, value: str, namespace_path: Optional[List[str]] = None
     ) -> None: ...
-    async def list_secrets(
-        self, namespace_path: Optional[List[str]] = None
-    ) -> List[str]: ...
+    def list_secrets(
+        self,
+        namespace_path: Optional[List[str]] = None,
+        *,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> NameListing: ...
     async def drop_secret(
         self, name: str, namespace_path: Optional[List[str]] = None
     ) -> None: ...
@@ -191,10 +199,16 @@ class Connection(object):
     async def drop_view_async(
         self, name: str, namespace_path: Optional[List[str]] = None
     ) -> Job: ...
-    async def list_views(
-        self, namespace_path: Optional[List[str]] = None
-    ) -> List[str]: ...
-    async def list_jobs(self) -> List[JobInfo]: ...
+    def list_views(
+        self,
+        namespace_path: Optional[List[str]] = None,
+        *,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> NameListing: ...
+    def list_jobs(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> JobListing: ...
     async def cancel_job(self, job_id: str) -> bool: ...
     async def pause_job(self, job_id: str) -> str: ...
     async def resume_job(self, job_id: str) -> str: ...
@@ -265,7 +279,9 @@ class Connection(object):
         limit: Optional[int] = None,
         with_no_data: bool = False,
     ) -> Job: ...
-    async def list_materialized_views(self) -> List[str]: ...
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> NameListing: ...
     async def drop_materialized_view(
         self, name: str, namespace_path: Optional[List[str]] = None
     ) -> None: ...
@@ -961,3 +977,21 @@ async def connect_catalog(
     read_consistency_interval: Optional[float] = None,
     oauth_config: Optional[Any] = None,
 ) -> Catalog: ...
+
+class NameListing:
+    def num_page_results(self) -> int: ...
+    def page_token(self) -> Optional[str]: ...
+    def __aiter__(self) -> NameListing: ...
+    async def __anext__(self) -> str: ...
+
+class FunctionListing:
+    def num_page_results(self) -> int: ...
+    def page_token(self) -> Optional[str]: ...
+    def __aiter__(self) -> FunctionListing: ...
+    async def __anext__(self) -> str: ...
+
+class JobListing:
+    def num_page_results(self) -> int: ...
+    def page_token(self) -> Optional[str]: ...
+    def __aiter__(self) -> JobListing: ...
+    async def __anext__(self) -> JobInfo: ...

python/python/lancedb/db.py

@@ -31,6 +31,7 @@ if sys.version_info >= (3, 12):
 else:
     from overrides import EnforceOverrides, override  # type: ignore
 
+from lancedb.listing import AsyncListing, Listing
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
 
 from lancedb.common import data_to_reader, sanitize_uri, validate_schema
@@ -668,11 +669,16 @@ class DBConnection(EnforceOverrides):
             "materialized views are not supported on this connection type"
         )
 
-    def list_materialized_views(self) -> List[str]:
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[str]:
         """The names of the materialized views in this database.
 
-        Found by reading every table's schema, so this costs an open per
-        table.
+        Local listings inspect table schemas one page at a time.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
         """
         raise NotImplementedError(
             "materialized views are not supported on this connection type"
@@ -885,8 +891,12 @@ class DBConnection(EnforceOverrides):
         )
 
     def list_functions(
-        self, *, namespace_path: Optional[List[str]] = None
-    ) -> List[FunctionVersion]:
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[FunctionVersion]:
         """List every published immutable Function version in a namespace.
 
         Functions in child namespaces are not included. Results are ordered by
@@ -900,6 +910,10 @@ class DBConnection(EnforceOverrides):
         ```python
         [(function.name, function.version) for function in db.list_functions()]
         ```
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
         """
         raise NotImplementedError(
             "Function catalog operations are not supported for this connection type"
@@ -971,11 +985,21 @@ class DBConnection(EnforceOverrides):
             "Secret operations are not supported for this connection type"
         )
 
-    def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
+    def list_secrets(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[str]:
         """The names of every Secret in this database.
 
         Names only. No method returns a stored credential, by construction
         rather than by policy. Local connections raise ``NotImplementedError``.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
         """
         raise NotImplementedError(
             "Secret operations are not supported for this connection type"
@@ -1073,12 +1097,22 @@ class DBConnection(EnforceOverrides):
             "View operations are not supported for this connection type"
         )
 
-    def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
+    def list_views(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[str]:
         """The names of the views in one namespace.
 
         Names only; a definition comes from
         [describe_view][lancedb.db.DBConnection.describe_view]. Local
         connections raise ``NotImplementedError``.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
         """
         raise NotImplementedError(
             "View operations are not supported for this connection type"
@@ -1097,8 +1131,15 @@ class DBConnection(EnforceOverrides):
         """
         raise NotImplementedError("open_job is not supported for this connection type")
 
-    def list_jobs(self) -> List[JobInfo]:
-        """List server-side jobs across the database's tables."""
+    def list_jobs(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[JobInfo]:
+        """List server-side jobs across the database's tables.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
         raise NotImplementedError("list_jobs is not supported for this connection type")
 
     def cancel_job(self, job_id: str) -> bool:
@@ -1699,9 +1740,20 @@ class LanceDBConnection(DBConnection):
         return view
 
     @override
-    def list_materialized_views(self) -> List[str]:
-        """The names of the materialized views in this database."""
-        return LOOP.run(self._conn.list_materialized_views())
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[str]:
+        """The names of the materialized views in this database.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_materialized_views(
+                page_token=page_token, page_limit=page_limit
+            )
+        )
 
     @override
     def drop_materialized_view(
@@ -1896,9 +1948,25 @@ class LanceDBConnection(DBConnection):
 
     @override
     def list_functions(
-        self, *, namespace_path: Optional[List[str]] = None
-    ) -> List[FunctionVersion]:
-        return LOOP.run(self._conn.list_functions(namespace_path=namespace_path))
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[FunctionVersion]:
+        """List functions.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_functions(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
 
     @override
     def drop_function(
@@ -1942,8 +2010,26 @@ class LanceDBConnection(DBConnection):
         LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
 
     @override
-    def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
-        return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
+    def list_secrets(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[str]:
+        """List secrets.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_secrets(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
 
     @override
     def drop_secret(
@@ -1986,13 +2072,40 @@ class LanceDBConnection(DBConnection):
         )
 
     @override
-    def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
-        return LOOP.run(self._conn.list_views(namespace_path=namespace_path))
+    def list_views(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[str]:
+        """List views.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_views(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
 
     @override
-    def list_jobs(self) -> List[JobInfo]:
-        """List server-side jobs across the database's tables."""
-        return LOOP.run(self._conn.list_jobs())
+    def list_jobs(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[JobInfo]:
+        """List server-side jobs across the database's tables.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
+        )
 
     @override
     def cancel_job(self, job_id: str) -> bool:
@@ -2680,13 +2793,22 @@ class AsyncConnection(object):
         await view.definition()
         return view
 
-    async def list_materialized_views(self) -> List[str]:
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> AsyncListing[str]:
         """The names of the materialized views in this database.
 
-        Found by reading every table's schema, so this costs an open per
-        table.
+        Local listings inspect table schemas one page at a time.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
         """
-        return await self._inner.list_materialized_views()
+        return AsyncListing(
+            self._inner.list_materialized_views(
+                page_token=page_token, page_limit=page_limit
+            )
+        )
 
     async def drop_materialized_view(
         self,
@@ -2908,19 +3030,31 @@ class AsyncConnection(object):
             await self._inner.get_function(name, version, namespace_path)
         )
 
-    async def list_functions(
-        self, *, namespace_path: Optional[List[str]] = None
-    ) -> List[FunctionVersion]:
+    def list_functions(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> AsyncListing[FunctionVersion]:
         """List every published immutable Function version in a namespace.
 
         Functions in child namespaces are not included. Results are ordered by
         Function name then version. Local connections raise
         ``NotImplementedError``.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
         """
-        return [
-            FunctionVersion.from_json(value)
-            for value in await self._inner.list_functions(namespace_path)
-        ]
+        return AsyncListing(
+            self._inner.list_functions(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            ),
+            FunctionVersion.from_json,
+        )
 
     async def drop_function(
         self,
@@ -2978,12 +3112,25 @@ class AsyncConnection(object):
             list(validate_namespace_path(namespace_path)),
         )
 
-    async def list_secrets(
-        self, *, namespace_path: Optional[List[str]] = None
-    ) -> List[str]:
-        """The names of every Secret in this database. Names only."""
-        return await self._inner.list_secrets(
-            list(validate_namespace_path(namespace_path))
+    def list_secrets(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> AsyncListing[str]:
+        """The names of every Secret in this database. Names only.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return AsyncListing(
+            self._inner.list_secrets(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
         )
 
     async def drop_secret(
@@ -3055,15 +3202,39 @@ class AsyncConnection(object):
             await self._inner.drop_view_async(name, namespace_path=namespace_path)
         )
 
-    async def list_views(
-        self, *, namespace_path: Optional[List[str]] = None
-    ) -> List[str]:
-        """The names of the views in one namespace."""
-        return await self._inner.list_views(list(namespace_path or []))
+    def list_views(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> AsyncListing[str]:
+        """The names of the views in one namespace.
 
-    async def list_jobs(self) -> List[JobInfo]:
-        """List server-side jobs across the database's tables."""
-        return await self._inner.list_jobs()
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return AsyncListing(
+            self._inner.list_views(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
+
+    def list_jobs(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> AsyncListing[JobInfo]:
+        """List server-side jobs across the database's tables.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return AsyncListing(
+            self._inner.list_jobs(page_token=page_token, page_limit=page_limit)
+        )
 
     async def cancel_job(self, job_id: str) -> bool:
         """Request cancellation of a server-side job by id.

python/python/lancedb/listing.py

@@ -0,0 +1,85 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Lazy, resumable resource listings."""
+
+from __future__ import annotations
+
+from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
+
+from .background_loop import LOOP
+
+T = TypeVar("T")
+
+
+class AsyncListing(AsyncIterator[T], Generic[T]):
+    """A lazy async iterator with inspectable pagination state.
+
+    Returned by connection resource-listing methods. For example::
+
+        async for name in connection.list_views(page_limit=20):
+            print(name)
+
+    Requests happen during iteration; request errors terminate the iterator.
+    """
+
+    def __init__(self, inner: Any, convert: Callable[[Any], T] = lambda value: value):
+        self._inner = inner
+        self._convert = convert
+
+    def __aiter__(self) -> AsyncListing[T]:
+        return self
+
+    async def __anext__(self) -> T:
+        return self._convert(await self._inner.__anext__())
+
+    def num_page_results(self) -> int:
+        """Return the number of cached items available without another request."""
+        return self._inner.num_page_results()
+
+    def page_token(self) -> Optional[str]:
+        """Return the token for the next request.
+
+        Initially this is the supplied starting token (``None`` starts at the
+        beginning). After fetching the final page it is ``None``, even if items
+        remain cached. Drain the cache before saving a token to avoid skipping
+        items when resuming. A failed request terminates iteration and retains
+        its token for retrying with a new iterator.
+        """
+        return self._inner.page_token()
+
+
+class Listing(Iterator[T], Generic[T]):
+    """A lazy synchronous iterator with inspectable pagination state.
+
+    For example::
+
+        for name in connection.list_views(page_limit=20):
+            print(name)
+
+    Use ``list(connection.list_views())`` to collect all results.
+    """
+
+    def __init__(self, inner: AsyncListing[T]):
+        self._inner = inner
+
+    def __iter__(self) -> Listing[T]:
+        return self
+
+    def __next__(self) -> T:
+        try:
+            return LOOP.run(self._inner.__anext__())
+        except StopAsyncIteration:
+            raise StopIteration from None
+
+    def num_page_results(self) -> int:
+        """Return the number of cached items available without another request."""
+        return self._inner.num_page_results()
+
+    def page_token(self) -> Optional[str]:
+        """Return the next request's token.
+
+        See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token]
+        for initial, cached, final-page, and error behavior.
+        """
+        return self._inner.page_token()

python/python/lancedb/namespace.py

@@ -11,7 +11,7 @@ through a namespace abstraction.
 from __future__ import annotations
 
 import sys
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
+from typing import Any, Dict, Iterable, List, Optional, Union
 from uuid import UUID
 
 if sys.version_info >= (3, 12):
@@ -19,8 +19,7 @@ if sys.version_info >= (3, 12):
 else:
     from overrides import override
 
-if TYPE_CHECKING:
-    from lancedb.query import Query
+from lancedb.query import Query
 
 from datetime import timedelta
 import pyarrow as pa
@@ -52,6 +51,7 @@ from lancedb.background_loop import LOOP
 from lancedb.arrow import AsyncRecordBatchReader
 from lancedb.db import AsyncConnection, DBConnection
 from lancedb.job import AsyncJob, Job
+from lancedb.listing import AsyncListing, Listing
 from lancedb.sql import AsyncQuery as AsyncSqlQuery
 from lancedb.sql import QueryDescription
 from lance_namespace import (
@@ -698,9 +698,20 @@ class LanceNamespaceDBConnection(DBConnection):
         return view
 
     @override
-    def list_materialized_views(self) -> List[str]:
-        """The names of the materialized views in the root namespace."""
-        return LOOP.run(self._inner.list_materialized_views())
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[str]:
+        """The names of the materialized views in the root namespace.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._inner.list_materialized_views(
+                page_token=page_token, page_limit=page_limit
+            )
+        )
 
     @override
     def drop_materialized_view(
@@ -1297,9 +1308,20 @@ class AsyncLanceNamespaceDBConnection:
         await view.definition()
         return view
 
-    async def list_materialized_views(self) -> List[str]:
-        """The names of the materialized views in the root namespace."""
-        return await self._inner.list_materialized_views()
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> AsyncListing[str]:
+        """The names of the materialized views in the root namespace.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return AsyncListing(
+            self._inner.list_materialized_views(
+                page_token=page_token, page_limit=page_limit
+            )
+        )
 
     async def drop_materialized_view(
         self, name: str, namespace_path: Optional[List[str]] = None

python/python/lancedb/remote/db.py

@@ -2,6 +2,8 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 
+from __future__ import annotations
+
 from dataclasses import replace
 from datetime import timedelta
 import json
@@ -34,6 +36,7 @@ from lancedb.remote import ClientConfig, RetryConfig, TimeoutConfig, TlsConfig
 import pyarrow as pa
 
 from ..common import DATA
+from lancedb.listing import Listing
 from ..db import DBConnection, LOOP
 from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job
@@ -769,8 +772,20 @@ class RemoteDBConnection(DBConnection):
         return view
 
     @override
-    def list_materialized_views(self) -> List[str]:
-        return LOOP.run(self._conn.list_materialized_views())
+    def list_materialized_views(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[str]:
+        """List materialized views.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_materialized_views(
+                page_token=page_token, page_limit=page_limit
+            )
+        )
 
     @override
     def drop_materialized_view(
@@ -885,9 +900,25 @@ class RemoteDBConnection(DBConnection):
 
     @override
     def list_functions(
-        self, *, namespace_path: Optional[List[str]] = None
-    ) -> List[FunctionVersion]:
-        return LOOP.run(self._conn.list_functions(namespace_path=namespace_path))
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[FunctionVersion]:
+        """List functions.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_functions(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
 
     @override
     def drop_function(
@@ -937,8 +968,26 @@ class RemoteDBConnection(DBConnection):
         return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
 
     @override
-    def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
-        return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
+    def list_secrets(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[str]:
+        """List secrets.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_secrets(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
 
     @override
     def drop_secret(
@@ -974,13 +1023,40 @@ class RemoteDBConnection(DBConnection):
         return Job(job)
 
     @override
-    def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
-        return LOOP.run(self._conn.list_views(namespace_path=namespace_path))
+    def list_views(
+        self,
+        *,
+        namespace_path: Optional[List[str]] = None,
+        page_token: Optional[str] = None,
+        page_limit: Optional[int] = None,
+    ) -> Listing[str]:
+        """List views.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_views(
+                namespace_path=namespace_path,
+                page_token=page_token,
+                page_limit=page_limit,
+            )
+        )
 
     @override
-    def list_jobs(self) -> List["JobInfo"]:
-        """List server-side jobs across the database's tables."""
-        return LOOP.run(self._conn.list_jobs())
+    def list_jobs(
+        self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+    ) -> Listing[JobInfo]:
+        """List server-side jobs across the database's tables.
+
+        Returns a lazy iterator. ``page_limit`` limits each request, not the total;
+        ``page_token`` resumes from a saved token. Requests and errors occur during
+        iteration. See [pagination state][lancedb.listing.AsyncListing].
+        """
+        return Listing(
+            self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
+        )
 
     @override
     def cancel_job(self, job_id: str) -> bool:

python/python/tests/test_first_class_function_slice2.py

@@ -1499,7 +1499,7 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
     with pytest.raises(NotImplementedError, match=message):
         db.get_function("normalize_score", version=FUNCTION_VERSION)
     with pytest.raises(NotImplementedError, match=message):
-        db.list_functions()
+        list(db.list_functions())
     with pytest.raises(NotImplementedError, match=message):
         db.drop_function("normalize_score", version=FUNCTION_VERSION)
 
@@ -1720,7 +1720,7 @@ def test_remote_secret_verbs_round_trip():
         )
         assert db.create_secret("openai-prod", "sk-live-0001") is None
         assert db.alter_secret("openai-prod", "sk-live-0002") is None
-        assert db.list_secrets() == ["openai-prod", "hf-prod"]
+        assert list(db.list_secrets()) == ["openai-prod", "hf-prod"]
         assert db.drop_secret("openai-prod") is None
 
     routes = [path for path, _ in state["requests"]]
@@ -1781,7 +1781,7 @@ def test_remote_list_functions_paginates_and_returns_typed_versions():
         )
         created = db.create_function(normalize_score)
         state["requests"].clear()
-        functions = db.list_functions()
+        functions = list(db.list_functions())
 
     assert functions == [created]
     assert state["requests"] == [
@@ -1805,7 +1805,7 @@ async def test_async_remote_list_functions_returns_typed_versions():
         registration = await db.create_function_async(normalize_score)
         created = await registration.wait()
         state["requests"].clear()
-        functions = await db.list_functions()
+        functions = [item async for item in db.list_functions()]
 
     assert functions == [created]
     assert [path for path, _ in state["requests"]] == [
@@ -1878,7 +1878,7 @@ def test_remote_function_verbs_address_a_namespace_in_the_path():
         assert reopened(value=lancedb.col("score")).function.namespace_path == tuple(
             namespace
         )
-        db.list_functions(namespace_path=namespace)
+        list(db.list_functions(namespace_path=namespace))
         assert (
             db.drop_function(
                 "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
@@ -1918,7 +1918,7 @@ async def test_async_remote_function_verbs_address_a_namespace_in_the_path():
         await db.get_function(
             "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
         )
-        await db.list_functions(namespace_path=namespace)
+        [item async for item in db.list_functions(namespace_path=namespace)]
         assert (
             await db.drop_function(
                 "normalize_score", version=FUNCTION_VERSION, namespace_path=namespace
@@ -1947,6 +1947,6 @@ def test_a_function_namespace_path_must_be_a_list_of_segments():
             client_config={"retry_config": {"retries": 0}},
         )
         with pytest.raises(TypeError):
-            db.list_functions(namespace_path="analytics")
+            list(db.list_functions(namespace_path="analytics"))
 
     assert state["requests"] == []

python/python/tests/test_listing.py

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import asyncio
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+import json
+from pathlib import Path
+from threading import Thread
+from urllib.parse import parse_qs, urlsplit
+
+import lancedb
+from lancedb.remote.errors import HttpError
+import pytest
+
+
+@pytest.fixture
+def listing_server():
+    requests, responses = [], []
+
+    class Handler(BaseHTTPRequestHandler):
+        def handle_request(self):
+            body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
+            requests.append((self.command, self.path, json.loads(body) if body else {}))
+            status, response = responses.pop(0)
+            self.send_response(status)
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            self.wfile.write(json.dumps(response).encode())
+
+        do_GET = handle_request
+        do_POST = handle_request
+
+    with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server:
+        thread = Thread(target=server.serve_forever, daemon=True)
+        thread.start()
+        try:
+            yield f"http://127.0.0.1:{server.server_port}", requests, responses
+        finally:
+            server.shutdown()
+            thread.join()
+
+
+def response_page(kind, names, token):
+    if kind == "secrets":
+        items = [{"name": name} for name in names]
+    elif kind == "jobs":
+        items = [
+            {
+                "job_id": name,
+                "table": "t",
+                "job_type": "create_index",
+                "state": "in_progress",
+                "created_at_millis": 1,
+            }
+            for name in names
+        ]
+    elif kind == "functions":
+        fixture = (
+            Path(__file__).resolve().parents[3]
+            / "rust/lancedb/tests/fixtures/first_class_functions/v1"
+            / "remote_function_version.canonical.json"
+        )
+        definition = json.loads(fixture.read_text())
+        items = [
+            {"name": "embed", "version": "1", "definition": definition} for _ in names
+        ]
+    else:
+        items = names
+    key = "views" if kind == "materialized_views" else kind
+    return {key: items, "page_token": token}
+
+
+async def connection(endpoint, asynchronous):
+    kwargs = dict(
+        api_key="fake",
+        host_override=endpoint,
+        client_config={"retry_config": {"retries": 0}},
+    )
+    if asynchronous:
+        return await lancedb.connect_async("db://dev", **kwargs)
+    return lancedb.connect("db://dev", **kwargs)
+
+
+async def advance(items, asynchronous):
+    return await items.__anext__() if asynchronous else next(items)
+
+
+KINDS = ["secrets", "views", "jobs", "functions", "materialized_views"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", KINDS)
+@pytest.mark.parametrize("asynchronous", [False, True])
+async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous):
+    endpoint, requests, responses = listing_server
+    responses.extend(
+        (200, response_page(kind, names, token))
+        for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], "")]
+    )
+    db = await connection(endpoint, asynchronous)
+    method = getattr(db, "list_" + kind)
+    kwargs = (
+        {} if kind in ("jobs", "materialized_views") else {"namespace_path": ["team"]}
+    )
+    items = method(page_token="start/token", page_limit=2, **kwargs)
+    assert requests == []
+    assert items.page_token() == "start/token"
+    assert items.num_page_results() == 0
+    first = await advance(items, asynchronous)
+    if kind == "functions":
+        assert first.name == "embed"
+        assert first.version == "1"
+    elif kind == "jobs":
+        assert first.job_id == "a"
+        assert first.state == "running"
+    else:
+        assert first == "a"
+    assert items.num_page_results() == 1
+    assert items.page_token() == "empty"
+    await advance(items, asynchronous)
+    assert items.num_page_results() == 0
+    assert len(requests) == 1
+    resumed = method(page_token=items.page_token(), page_limit=2, **kwargs)
+    await advance(resumed, asynchronous)
+    assert len(requests) == 3
+    assert resumed.page_token() is None
+    assert resumed.num_page_results() == 1
+    remaining = [item async for item in resumed] if asynchronous else list(resumed)
+    assert len(remaining) == 1
+    assert resumed.num_page_results() == 0
+    assert ([item async for item in resumed] if asynchronous else list(resumed)) == []
+    for (verb, url, body), token in zip(requests, ["start/token", "empty", "last"]):
+        path = urlsplit(url)
+        if kind == "jobs":
+            assert verb == "POST"
+            assert path.path == "/v1/jobs/list"
+            assert body == {"page_token": token, "limit": 2}
+        else:
+            namespace = "$" if kind == "materialized_views" else "team"
+            resource = kind.removesuffix("s")
+            assert path.path == f"/v1/namespace/{namespace}/{resource}/list"
+            expected = {"page_token": [token], "limit": ["2"]}
+            if kind == "functions":
+                expected["include_definition"] = ["true"]
+            assert parse_qs(path.query) == expected
+    assert responses == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", KINDS)
+@pytest.mark.parametrize("asynchronous", [False, True])
+async def test_listing_error_retains_token(listing_server, kind, asynchronous):
+    endpoint, requests, responses = listing_server
+    responses.extend([(401, {}), (200, response_page(kind, ["a"], None))])
+    db = await connection(endpoint, asynchronous)
+    method = getattr(db, "list_" + kind)
+    items = method(page_token="retry")
+    with pytest.raises(HttpError):
+        await advance(items, asynchronous)
+    assert items.page_token() == "retry"
+    assert items.num_page_results() == 0
+    assert ([item async for item in items] if asynchronous else list(items)) == []
+    assert len(requests) == 1
+    resumed = method(page_token=items.page_token())
+    assert len([item async for item in resumed] if asynchronous else list(resumed)) == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", KINDS)
+async def test_listing_serializes_concurrent_advances(listing_server, kind):
+    endpoint, requests, responses = listing_server
+    responses.append((200, response_page(kind, ["a", "b"], None)))
+    db = await connection(endpoint, True)
+    items = getattr(db, "list_" + kind)()
+    results = await asyncio.gather(
+        items.__anext__(), items.__anext__(), items.__anext__(), return_exceptions=True
+    )
+    assert (
+        len([result for result in results if isinstance(result, StopAsyncIteration)])
+        == 1
+    )
+    assert len(requests) == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", KINDS)
+@pytest.mark.parametrize("page_limit", [0, 2**32 - 1])
+async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_limit):
+    endpoint, requests, _ = listing_server
+    db = await connection(endpoint, True)
+    items = getattr(db, "list_" + kind)(page_limit=page_limit)
+    with pytest.raises(ValueError, match="limit"):
+        await items.__anext__()
+    assert requests == []

python/python/tests/test_materialized_views.py

@@ -93,7 +93,7 @@ def test_remote_list_uses_namespace_route():
             host_override=host,
             client_config={"retry_config": {"retries": 0}},
         )
-        assert db.list_materialized_views() == ["daily_sales"]
+        assert list(db.list_materialized_views()) == ["daily_sales"]
     assert requests == ["/v1/namespace/$/materialized_view/list"]
 
 
@@ -190,7 +190,7 @@ def test_create_and_refresh_jobs(tmp_path):
     drop_job = db.drop_materialized_view_async("adults")
     assert drop_job.id is None
     assert drop_job.wait() is None
-    assert "adults" not in db.list_materialized_views()
+    assert "adults" not in list(db.list_materialized_views())
 
 
 def test_definition_round_trips(tmp_path):
@@ -250,14 +250,14 @@ def test_list_and_not_a_view(tmp_path):
     db = make_db(tmp_path)
     db.create_materialized_view("adults", "people", where="age >= 18")
 
-    assert db.list_materialized_views() == ["adults"]
+    assert list(db.list_materialized_views()) == ["adults"]
     with pytest.raises(ValueError, match="not a materialized view"):
         db.open_materialized_view("people")
     with pytest.raises(ValueError, match="not a materialized view"):
         db.drop_materialized_view("people")
 
     db.drop_materialized_view("adults")
-    assert db.list_materialized_views() == []
+    assert list(db.list_materialized_views()) == []
 
 
 def test_invalid_expression_fails_at_create(tmp_path):
@@ -285,7 +285,7 @@ async def test_async_create_refresh_and_open(tmp_path):
     reopened = await db.open_materialized_view("shouts")
     definition = await reopened.definition()
     assert definition.query == "SELECT upper(name) AS shout FROM people"
-    assert await db.list_materialized_views() == ["shouts"]
+    assert [item async for item in db.list_materialized_views()] == ["shouts"]
 
 
 @pytest.mark.asyncio
@@ -309,7 +309,7 @@ async def test_async_create_and_refresh_jobs(tmp_path):
     drop_job = await db.drop_materialized_view_async("adults")
     assert drop_job.id is None
     assert await drop_job.wait() is None
-    assert "adults" not in await db.list_materialized_views()
+    assert "adults" not in [item async for item in db.list_materialized_views()]
 
 
 @pytest.mark.asyncio
@@ -399,7 +399,7 @@ def test_namespace_connection_materialized_views(tmp_path):
     view = db.create_materialized_view("adults", "people", where="age >= 18")
     view.refresh()
     assert view.table.count_rows() == 1
-    assert db.list_materialized_views() == ["adults"]
+    assert list(db.list_materialized_views()) == ["adults"]
 
     reopened = db.open_materialized_view("adults")
     assert reopened.definition.query.startswith("SELECT name, age FROM ")
@@ -415,7 +415,7 @@ def test_namespace_connection_materialized_views(tmp_path):
 
     assert db.drop_materialized_view_async("job_view").wait() is None
     db.drop_materialized_view("adults")
-    assert db.list_materialized_views() == []
+    assert list(db.list_materialized_views()) == []
 
 
 @pytest.mark.asyncio
@@ -434,7 +434,7 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
     view = await db.create_materialized_view("adults", "people", where="age >= 18")
     await view.refresh()
     assert await view.table.count_rows() == 1
-    assert await db.list_materialized_views() == ["adults"]
+    assert [item async for item in db.list_materialized_views()] == ["adults"]
 
     reopened = await db.open_materialized_view("adults")
     assert (await reopened.definition()).query.startswith("SELECT name, age FROM ")
@@ -460,7 +460,7 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
     drop_job = await db.drop_materialized_view_async("job_view")
     assert await drop_job.wait() is None
     await db.drop_materialized_view("adults")
-    assert await db.list_materialized_views() == []
+    assert [item async for item in db.list_materialized_views()] == []
 
 
 def test_stored_queries_and_legacy_layouts_are_read():
@@ -509,3 +509,19 @@ def test_stored_queries_and_legacy_layouts_are_read():
     ):
         with pytest.raises(NotImplementedError, match="cannot refresh"):
             read(newer)
+
+
+def test_local_materialized_view_listing_pages_and_resume(tmp_path):
+    db = make_db(tmp_path)
+    db.create_materialized_view("a_view", "people", with_no_data=True)
+    db.create_materialized_view("z_view", "people", with_no_data=True)
+    db.create_table("ordinary", [{"id": 1}])
+    names = db.list_materialized_views(page_limit=1)
+    assert names.num_page_results() == 0
+    first = next(names)
+    assert names.num_page_results() == 0
+    assert names.page_token() is not None
+    resumed = db.list_materialized_views(page_token=names.page_token(), page_limit=1)
+    assert sorted([first, *resumed]) == ["a_view", "z_view"]
+    assert resumed.page_token() is None
+    assert list(resumed) == []

python/python/tests/test_namespace.py

@@ -10,6 +10,7 @@ import pytest
 import pyarrow as pa
 import lancedb
 from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
+from lancedb.listing import AsyncListing, Listing
 from lancedb.namespace import _MAX_QUERY_K
 from lancedb.table import AsyncTable, LanceTable
 
@@ -104,6 +105,18 @@ class TestNamespaceConnection:
         assert isinstance(db, lancedb.LanceNamespaceDBConnection)
         assert len(list(db.table_names())) == 0
 
+    def test_list_materialized_views(self):
+        db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
+        db.create_table("source", [{"id": 1}])
+        db.create_materialized_view("view", "source", with_no_data=True)
+
+        names = db.list_materialized_views(page_limit=1)
+        assert isinstance(names, Listing)
+        assert names.num_page_results() == 0
+        assert list(names) == ["view"]
+        assert names.num_page_results() == 0
+        assert names.page_token() is None
+
     def test_sync_builtin_namespace_uses_rust_without_python_client(self, monkeypatch):
         """Built-in sync namespace connections should not construct or call the
         Python namespace client for normal namespace/table management."""
@@ -605,6 +618,18 @@ class TestAsyncNamespaceConnection:
         table_names = await db.table_names()
         assert len(list(table_names)) == 0
 
+    async def test_list_materialized_views(self):
+        db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
+        await db.create_table("source", [{"id": 1}])
+        await db.create_materialized_view("view", "source", with_no_data=True)
+
+        names = db.list_materialized_views(page_limit=1)
+        assert isinstance(names, AsyncListing)
+        assert names.num_page_results() == 0
+        assert [name async for name in names] == ["view"]
+        assert names.num_page_results() == 0
+        assert names.page_token() is None
+
     async def test_async_builtin_namespace_uses_rust_without_python_client(
         self, monkeypatch
     ):

python/python/tests/test_remote_db.py

@@ -3125,7 +3125,7 @@ def test_remote_connection_jobs_surface():
             request.end_headers()
 
     with mock_lancedb_connection(handler) as db:
-        jobs = db.list_jobs()
+        jobs = list(db.list_jobs())
         assert [job.job_id for job in jobs] == ["job-1", "job-2"]
         assert jobs[0].state == "running"
         assert jobs[0].table == "t1"
@@ -3285,7 +3285,7 @@ def test_view_crud_addresses_its_own_routes():
         described = db.describe_view("adults", namespace_path=["analytics"])
         assert described.schema == view.schema
 
-        assert db.list_views(namespace_path=["analytics"]) == ["adults"]
+        assert list(db.list_views(namespace_path=["analytics"])) == ["adults"]
         db.drop_view("adults", namespace_path=["analytics"])
 
     assert paths == [

python/src/connection.rs

@@ -464,12 +464,16 @@ impl Connection {
         })
     }
 
-    pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            let views = inner.list_materialized_views().await.infer_error()?;
-            Ok(views)
-        })
+    #[pyo3(signature = (*, page_token=None, page_limit=None))]
+    pub fn list_materialized_views(
+        &self,
+        page_token: Option<String>,
+        page_limit: Option<u32>,
+    ) -> PyResult<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?
+                .list_materialized_views(crate::listing::options(page_token, page_limit)),
+        ))
     }
 
     #[pyo3(signature = (name, namespace_path=None))]
@@ -776,22 +780,19 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (namespace_path=None))]
+    #[pyo3(signature = (namespace_path=None, *, page_token=None, page_limit=None))]
     pub fn list_functions(
-        self_: PyRef<'_, Self>,
+        &self,
         namespace_path: Option<Vec<String>>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let namespace_path = namespace_path.unwrap_or_default();
-        future_into_py(self_.py(), async move {
-            inner
-                .list_functions(&namespace_path)
-                .await
-                .infer_error()?
-                .into_iter()
-                .map(|function| function.to_canonical_json().infer_error())
-                .collect::<PyResult<Vec<_>>>()
-        })
+        page_token: Option<String>,
+        page_limit: Option<u32>,
+    ) -> PyResult<crate::listing::FunctionListing> {
+        Ok(crate::listing::FunctionListing::new(
+            self.get_inner()?.list_functions(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, page_limit),
+            ),
+        ))
     }
 
     #[pyo3(signature = (name, version, namespace_path=None))]
@@ -863,16 +864,19 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (namespace_path=None))]
+    #[pyo3(signature = (namespace_path=None, *, page_token=None, page_limit=None))]
     pub fn list_secrets(
-        self_: PyRef<'_, Self>,
+        &self,
         namespace_path: Option<Vec<String>>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let namespace_path = namespace_path.unwrap_or_default();
-        future_into_py(self_.py(), async move {
-            inner.list_secrets(&namespace_path).await.infer_error()
-        })
+        page_token: Option<String>,
+        page_limit: Option<u32>,
+    ) -> PyResult<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?.list_secrets(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, page_limit),
+            ),
+        ))
     }
 
     #[pyo3(signature = (name, namespace_path=None))]
@@ -973,27 +977,31 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (namespace_path=None))]
+    #[pyo3(signature = (namespace_path=None, *, page_token=None, page_limit=None))]
     pub fn list_views(
-        self_: PyRef<'_, Self>,
+        &self,
         namespace_path: Option<Vec<String>>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let namespace_path = namespace_path.unwrap_or_default();
-        future_into_py(self_.py(), async move {
-            inner.list_views(&namespace_path).await.infer_error()
-        })
+        page_token: Option<String>,
+        page_limit: Option<u32>,
+    ) -> PyResult<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?.list_views(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, page_limit),
+            ),
+        ))
     }
 
-    pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        future_into_py(self_.py(), async move {
-            let jobs = inner.list_jobs().await.infer_error()?;
-            Ok(jobs
-                .into_iter()
-                .map(crate::job::JobInfo::from)
-                .collect::<Vec<_>>())
-        })
+    #[pyo3(signature = (*, page_token=None, page_limit=None))]
+    pub fn list_jobs(
+        &self,
+        page_token: Option<String>,
+        page_limit: Option<u32>,
+    ) -> PyResult<crate::listing::JobListing> {
+        Ok(crate::listing::JobListing::new(
+            self.get_inner()?
+                .list_jobs(crate::listing::options(page_token, page_limit)),
+        ))
     }
 
     pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {

python/src/lib.rs

@@ -29,6 +29,7 @@ pub mod expr;
 pub mod header;
 pub mod index;
 pub mod job;
+mod listing;
 pub mod namespace;
 pub mod oauth;
 pub mod otel;
@@ -65,6 +66,10 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<Connection>()?;
     m.add_class::<catalog::Catalog>()?;
     m.add_class::<catalog::DatabaseNames>()?;
+    m.add_class::<listing::NameListing>()?;
+    m.add_class::<listing::FunctionListing>()?;
+    m.add_class::<listing::JobListing>()?;
+
     m.add_function(wrap_pyfunction!(catalog::connect_catalog, m)?)?;
     m.add_class::<Session>()?;
     m.add_class::<Table>()?;

python/src/listing.rs

@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use crate::error::PythonErrorExt;
+use crate::runtime::future_into_py;
+use futures::{StreamExt, future::poll_fn};
+use lancedb::listing::{Listing, ListingOptions};
+use pyo3::exceptions::PyStopAsyncIteration;
+use pyo3::{Bound, PyAny, PyRef, PyResult, Python, pyclass, pymethods};
+use std::sync::{Arc, Mutex};
+
+pub fn options(page_token: Option<String>, page_limit: Option<u32>) -> ListingOptions {
+    let mut options = ListingOptions::default();
+    options.page_token = page_token;
+    options.page_limit = page_limit;
+    options
+}
+
+#[pyclass]
+pub struct NameListing {
+    inner: Arc<Mutex<Listing<String>>>,
+    next_lock: Arc<tokio::sync::Mutex<()>>,
+}
+impl NameListing {
+    pub fn new(inner: Listing<String>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(inner)),
+            next_lock: Arc::new(tokio::sync::Mutex::new(())),
+        }
+    }
+}
+#[pymethods]
+impl NameListing {
+    fn num_page_results(&self, py: Python<'_>) -> usize {
+        py.detach(|| self.inner.lock().unwrap().num_page_results())
+    }
+    fn page_token(&self, py: Python<'_>) -> Option<String> {
+        py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
+    }
+    fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        self_
+    }
+    fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        let next_lock = self_.next_lock.clone();
+        future_into_py(self_.py(), async move {
+            let _guard = next_lock.lock().await;
+            let value = poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
+                .await
+                .ok_or_else(|| PyStopAsyncIteration::new_err(""))?
+                .infer_error()?;
+            Ok(value)
+        })
+    }
+}
+
+#[pyclass]
+pub struct FunctionListing {
+    inner: Arc<Mutex<Listing<lancedb::function::FunctionVersion>>>,
+    next_lock: Arc<tokio::sync::Mutex<()>>,
+}
+impl FunctionListing {
+    pub fn new(inner: Listing<lancedb::function::FunctionVersion>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(inner)),
+            next_lock: Arc::new(tokio::sync::Mutex::new(())),
+        }
+    }
+}
+#[pymethods]
+impl FunctionListing {
+    fn num_page_results(&self, py: Python<'_>) -> usize {
+        py.detach(|| self.inner.lock().unwrap().num_page_results())
+    }
+    fn page_token(&self, py: Python<'_>) -> Option<String> {
+        py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
+    }
+    fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        self_
+    }
+    fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        let next_lock = self_.next_lock.clone();
+        future_into_py(self_.py(), async move {
+            let _guard = next_lock.lock().await;
+            let value = poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
+                .await
+                .ok_or_else(|| PyStopAsyncIteration::new_err(""))?
+                .infer_error()?;
+            value.to_canonical_json().infer_error()
+        })
+    }
+}
+
+#[pyclass]
+pub struct JobListing {
+    inner: Arc<Mutex<Listing<lancedb::database::JobInfo>>>,
+    next_lock: Arc<tokio::sync::Mutex<()>>,
+}
+impl JobListing {
+    pub fn new(inner: Listing<lancedb::database::JobInfo>) -> Self {
+        Self {
+            inner: Arc::new(Mutex::new(inner)),
+            next_lock: Arc::new(tokio::sync::Mutex::new(())),
+        }
+    }
+}
+#[pymethods]
+impl JobListing {
+    fn num_page_results(&self, py: Python<'_>) -> usize {
+        py.detach(|| self.inner.lock().unwrap().num_page_results())
+    }
+    fn page_token(&self, py: Python<'_>) -> Option<String> {
+        py.detach(|| self.inner.lock().unwrap().page_token().map(str::to_owned))
+    }
+    fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        self_
+    }
+    fn __anext__(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        let next_lock = self_.next_lock.clone();
+        future_into_py(self_.py(), async move {
+            let _guard = next_lock.lock().await;
+            let value = poll_fn(|cx| inner.lock().unwrap().poll_next_unpin(cx))
+                .await
+                .ok_or_else(|| PyStopAsyncIteration::new_err(""))?
+                .infer_error()?;
+            Ok(crate::job::JobInfo::from(value))
+        })
+    }
+}