diff --git a/docs/web/basics/jobs.mdx b/docs/web/basics/jobs.mdx
index 264acd728..db5b49ab5 100644
--- a/docs/web/basics/jobs.mdx
+++ b/docs/web/basics/jobs.mdx
@@ -177,14 +177,51 @@ for info in db.list_jobs(page_limit=50):
live `Job` handles. Open an ID to inspect its full record. `page_limit` bounds
each server request, not the total yielded items, and `page_token` resumes a
listing. It is a lazy iterator: iteration performs requests and can raise
-request errors. To materialize it, use `list(db.list_jobs(...))`; it does not
-support list indexing, `len()`, or a `.to_list()` method.
+request errors. Use `list(db.list_jobs(...))` to collect all remaining results;
+there is no `.to_list()` method.
-`Listing.num_page_results()` reports cached items remaining in the current
-page and `Listing.page_token()` reports the next request's token. Drain the
-cached items before saving that token, or resuming can skip them. A `None`
-token after the final page can coexist with unread cached items. A resumed
-listing is not a transactionally frozen view of jobs submitted in the meantime.
+
+Job filters and synchronous `Listing` indexing, slicing, and `len()` require
+a Python SDK release that includes these APIs. Older SDKs expose only
+`page_token` and `page_limit`; collect results with `list(...)` before indexing
+or counting them. Filtering also requires a matching Enterprise server that
+supports the requested filters.
+
+
+Filter jobs on the server before pagination with `table`, `state`, `job_type`
+or `job_types`, and creation-time bounds. `table` names a table in
+`namespace_path` (the root by default) and includes all its branches. Requests
+stay scoped to the connected database. Creation times are Unix milliseconds:
+`created_at_gte_millis` is inclusive and `created_at_lt_millis` is exclusive.
+Choose either `job_type` or `job_types`.
+
+The `state` filter uses registry states: `in_progress`, `succeeded`, `failed`,
+`canceled`, and `timed_out`. These differ from the SDK lifecycle labels such
+as `running` and `finished` returned in `JobInfo.state`.
+
+```python
+jobs = db.list_jobs(table="events", state="in_progress", page_limit=5)
+first_five = jobs[:5]
+for info in first_five:
+ print(info.job_id, info.state)
+```
+
+Synchronous indexing and slicing refer to the remaining results without
+consuming them. A bounded forward slice such as `jobs[:5]` fetches only enough
+pages to reach its stop. `len(jobs)` returns the remaining count by fetching
+and caching every remaining result. Negative indexes and open-ended or reverse
+slices also fetch and cache all remaining results. These operations can raise
+request errors; later iteration drains their cache. `AsyncListing` remains an
+async iterator: use `async for` to read it.
+
+`Listing.num_page_results()` reports cached items available without another
+request, including items retained by indexing, slicing, or counting.
+`Listing.page_token()` reports the next request's token. Drain all cached
+items before saving that token, or resuming can skip them. Resume with the same
+filters, for example `db.list_jobs(table="events", state="in_progress",
+page_token=saved_token, page_limit=5)`. A `None` token after the final page can
+coexist with unread cached items. A resumed listing is not a transactionally
+frozen view of jobs submitted in the meantime.
`events.schema` describes the Arrow response. Use `events.to_pylist()` for
Python row dictionaries. Common event columns are `event_seq` (int64), `state`
diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts
index 215df992d..9baa51292 100644
--- a/nodejs/__test__/remote.test.ts
+++ b/nodejs/__test__/remote.test.ts
@@ -2318,10 +2318,30 @@ describe("remote connection jobs surface", () => {
queue_reason: "Waiting for Function worker capacity",
});
const descriptions = [
- `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 3000, unknownFleet)}}`,
- `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 2000, knownFleet)}}`,
- `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(40, 60, 1000, unknownFleet)}}`,
- `{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(60, 0, 0, unknownFleet)}}`,
+ `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
+ 20,
+ 80,
+ 3000,
+ unknownFleet,
+ )}}`,
+ `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
+ 20,
+ 80,
+ 2000,
+ knownFleet,
+ )}}`,
+ `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
+ 40,
+ 60,
+ 1000,
+ unknownFleet,
+ )}}`,
+ `{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(
+ 60,
+ 0,
+ 0,
+ unknownFleet,
+ )}}`,
];
let describeCount = 0;
@@ -2393,12 +2413,24 @@ describe("remote connection jobs surface", () => {
it("does not derive ETA across a zero-rate, epoch, or pause boundary", async () => {
const now = Date.now();
const descriptions = [
- `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 5000}}}`,
- `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 4000}}}`,
- `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 3000}}}`,
- `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 2000}}}`,
- `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1500}}}`,
- `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1000}}}`,
+ `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${
+ now - 5000
+ }}}`,
+ `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${
+ now - 4000
+ }}}`,
+ `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
+ now - 3000
+ }}}`,
+ `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
+ now - 2000
+ }}}`,
+ `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
+ now - 1500
+ }}}`,
+ `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
+ now - 1000
+ }}}`,
];
await withMockDatabase(
@@ -2574,6 +2606,7 @@ describe("lazy resource listings", () => {
if (kind === "jobs") {
expect(url.pathname).toBe("/v1/jobs/list");
expect(request.body).toEqual({
+ database: "dev",
// biome-ignore lint/style/useNamingConvention: server wire format
page_token: token,
limit: 2,
diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi
index a86bae9d5..0cce2085e 100644
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -300,7 +300,19 @@ class Connection(object):
page_limit: Optional[int] = None,
) -> NameListing: ...
def list_jobs(
- self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+ self,
+ *,
+ page_token: Optional[str] = None,
+ page_limit: Optional[int] = None,
+ table: Optional[str] = None,
+ namespace_path: Optional[List[str]] = None,
+ state: Optional[
+ Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
+ ] = None,
+ job_type: Optional[str] = None,
+ job_types: Optional[List[str]] = None,
+ created_at_gte_millis: Optional[int] = None,
+ created_at_lt_millis: Optional[int] = None,
) -> JobListing: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def pause_job(self, job_id: str) -> str: ...
diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py
index 12c7edd5c..b225a6e65 100644
--- a/python/python/lancedb/db.py
+++ b/python/python/lancedb/db.py
@@ -1602,13 +1602,39 @@ class DBConnection(EnforceOverrides):
)
def list_jobs(
- self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+ self,
+ *,
+ page_token: Optional[str] = None,
+ page_limit: Optional[int] = None,
+ table: Optional[str] = None,
+ namespace_path: Optional[List[str]] = None,
+ state: Optional[
+ Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
+ ] = None,
+ job_type: Optional[str] = None,
+ job_types: Optional[List[str]] = None,
+ created_at_gte_millis: Optional[int] = None,
+ created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
- """List server-side jobs across the database's tables.
+ """List server-side jobs, optionally filtered before pagination.
- Returns a lazy iterator. ``page_limit`` limits each request, not the total;
- ``page_token`` resumes from a saved token. Requests and errors occur during
- iteration. See [pagination state][lancedb.listing.AsyncListing].
+ ``table`` names a table in ``namespace_path`` (the root by default).
+ Listings include all branches of the table. A namespace filter requires
+ ``table``.
+ ``state`` uses registry states such as ``in_progress``, ``succeeded``,
+ ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
+ ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
+ Creation-time bounds are Unix milliseconds, inclusive at
+ ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
+
+ Returns a lazy iterator. ``page_limit`` limits each request, not the total.
+ Resume a ``page_token`` with the same filters. Requests and errors occur
+ during iteration. See [pagination state][lancedb.listing.AsyncListing].
+
+ For example, to inspect up to five jobs without fetching all history::
+
+ jobs = db.list_jobs(table="events", page_limit=5)
+ first_five = jobs[:5]
"""
raise NotImplementedError(
_append_doc_url(
@@ -2799,16 +2825,52 @@ class LanceDBConnection(DBConnection):
@override
def list_jobs(
- self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+ self,
+ *,
+ page_token: Optional[str] = None,
+ page_limit: Optional[int] = None,
+ table: Optional[str] = None,
+ namespace_path: Optional[List[str]] = None,
+ state: Optional[
+ Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
+ ] = None,
+ job_type: Optional[str] = None,
+ job_types: Optional[List[str]] = None,
+ created_at_gte_millis: Optional[int] = None,
+ created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
- """List server-side jobs across the database's tables.
+ """List server-side jobs, optionally filtered before pagination.
- Returns a lazy iterator. ``page_limit`` limits each request, not the total;
- ``page_token`` resumes from a saved token. Requests and errors occur during
- iteration. See [pagination state][lancedb.listing.AsyncListing].
+ ``table`` names a table in ``namespace_path`` (the root by default).
+ Listings include all branches of the table. A namespace filter requires
+ ``table``.
+ ``state`` uses registry states such as ``in_progress``, ``succeeded``,
+ ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
+ ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
+ Creation-time bounds are Unix milliseconds, inclusive at
+ ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
+
+ Returns a lazy iterator. ``page_limit`` limits each request, not the total.
+ Resume a ``page_token`` with the same filters. Requests and errors occur
+ during iteration. See [pagination state][lancedb.listing.AsyncListing].
+
+ For example, to inspect up to five jobs without fetching all history::
+
+ jobs = db.list_jobs(table="events", page_limit=5)
+ first_five = jobs[:5]
"""
return Listing(
- self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
+ self._conn.list_jobs(
+ page_token=page_token,
+ page_limit=page_limit,
+ table=table,
+ namespace_path=namespace_path,
+ state=state,
+ job_type=job_type,
+ job_types=job_types,
+ created_at_gte_millis=created_at_gte_millis,
+ created_at_lt_millis=created_at_lt_millis,
+ )
)
@override
@@ -4375,16 +4437,56 @@ class AsyncConnection(object):
)
def list_jobs(
- self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+ self,
+ *,
+ page_token: Optional[str] = None,
+ page_limit: Optional[int] = None,
+ table: Optional[str] = None,
+ namespace_path: Optional[List[str]] = None,
+ state: Optional[
+ Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
+ ] = None,
+ job_type: Optional[str] = None,
+ job_types: Optional[List[str]] = None,
+ created_at_gte_millis: Optional[int] = None,
+ created_at_lt_millis: Optional[int] = None,
) -> AsyncListing[JobInfo]:
- """List server-side jobs across the database's tables.
+ """List server-side jobs, optionally filtered before pagination.
- Returns a lazy iterator. ``page_limit`` limits each request, not the total;
- ``page_token`` resumes from a saved token. Requests and errors occur during
- iteration. See [pagination state][lancedb.listing.AsyncListing].
+ ``table`` names a table in ``namespace_path`` (the root by default).
+ Listings include all branches of the table. A namespace filter requires
+ ``table``.
+ ``state`` uses registry states such as ``in_progress``, ``succeeded``,
+ ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
+ ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
+ Creation-time bounds are Unix milliseconds, inclusive at
+ ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
+
+ Returns a lazy iterator. ``page_limit`` limits each request, not the total.
+ Resume a ``page_token`` with the same filters. Requests and errors occur
+ during iteration. See [pagination state][lancedb.listing.AsyncListing].
+
+ For example, to inspect up to five jobs without fetching all history::
+
+ jobs = db.list_jobs(table="events", page_limit=5)
+ first_five = []
+ async for job in jobs:
+ first_five.append(job)
+ if len(first_five) == 5:
+ break
"""
return AsyncListing(
- self._inner.list_jobs(page_token=page_token, page_limit=page_limit)
+ self._inner.list_jobs(
+ page_token=page_token,
+ page_limit=page_limit,
+ table=table,
+ namespace_path=namespace_path,
+ state=state,
+ job_type=job_type,
+ job_types=job_types,
+ created_at_gte_millis=created_at_gte_millis,
+ created_at_lt_millis=created_at_lt_millis,
+ )
)
async def cancel_job(self, job_id: str) -> bool:
diff --git a/python/python/lancedb/listing.py b/python/python/lancedb/listing.py
index 59aba1431..f85599b2d 100644
--- a/python/python/lancedb/listing.py
+++ b/python/python/lancedb/listing.py
@@ -6,7 +6,19 @@
from __future__ import annotations
import asyncio
-from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
+from collections import deque
+from itertools import islice
+from operator import index
+from typing import (
+ Any,
+ AsyncIterator,
+ Callable,
+ Generic,
+ Iterator,
+ Optional,
+ TypeVar,
+ overload,
+)
from .background_loop import LOOP
@@ -59,29 +71,85 @@ class Listing(Iterator[T], Generic[T]):
print(name)
Use ``list(connection.list_views())`` to collect all results.
+
+ Indexes and slices refer to the remaining items and do not consume them.
+ A bounded forward slice, such as ``listing[:5]``, fetches only enough pages
+ to reach its stop. ``len(listing)``, negative indexes, and open-ended or
+ reverse slices fetch and cache all remaining results. Iteration then drains
+ that cache. These operations may perform requests and raise request errors.
"""
def __init__(self, inner: AsyncListing[T]):
self._inner = inner
+ self._buffer: deque[T] = deque()
def __iter__(self) -> Listing[T]:
return self
def __next__(self) -> T:
+ if self._buffer:
+ return self._buffer.popleft()
try:
return LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
raise StopIteration from None
+ def _fill(self, count: Optional[int]) -> None:
+ while count is None or len(self._buffer) < count:
+ try:
+ item = LOOP.run(self._inner.__anext__())
+ except StopAsyncIteration:
+ return
+ self._buffer.append(item)
+
+ def __len__(self) -> int:
+ """Fetch and cache every remaining item, returning the remaining count."""
+ self._fill(None)
+ return len(self._buffer)
+
+ def __bool__(self) -> bool:
+ self._fill(1)
+ return bool(self._buffer)
+
+ @overload
+ def __getitem__(self, key: int) -> T: ...
+
+ @overload
+ def __getitem__(self, key: slice) -> list[T]: ...
+
+ def __getitem__(self, key: int | slice) -> T | list[T]:
+ """Read remaining items without consuming them; bounded slices stay lazy."""
+ if isinstance(key, slice):
+ start = 0 if key.start is None else index(key.start)
+ stop = None if key.stop is None else index(key.stop)
+ step = 1 if key.step is None else index(key.step)
+ if step == 0:
+ raise ValueError("slice step cannot be zero")
+ if start >= 0 and stop is not None and stop >= 0 and step > 0:
+ if start >= stop:
+ return []
+ self._fill(stop)
+ start, stop, step = key.indices(len(self._buffer))
+ return list(
+ islice(self._buffer, start, stop, min(step, len(self._buffer) + 1))
+ )
+ self._fill(None)
+ return list(self._buffer)[key]
+ position = index(key)
+ self._fill(None if position < 0 else position + 1)
+ return self._buffer[position]
+
def num_page_results(self) -> int:
"""Return the number of cached items available without another request."""
- return self._inner.num_page_results()
+ return len(self._buffer) + self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the next request's token.
See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token]
for initial, cached, final-page, and error behavior.
+ Indexing, slicing, and counting can also advance this token; drain all
+ ``num_page_results()`` cached items before resuming from it.
"""
return self._inner.page_token()
diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py
index 913c8fc2c..2927f2726 100644
--- a/python/python/lancedb/remote/db.py
+++ b/python/python/lancedb/remote/db.py
@@ -1324,16 +1324,52 @@ class RemoteDBConnection(DBConnection):
@override
def list_jobs(
- self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+ self,
+ *,
+ page_token: Optional[str] = None,
+ page_limit: Optional[int] = None,
+ table: Optional[str] = None,
+ namespace_path: Optional[List[str]] = None,
+ state: Optional[
+ Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
+ ] = None,
+ job_type: Optional[str] = None,
+ job_types: Optional[List[str]] = None,
+ created_at_gte_millis: Optional[int] = None,
+ created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
- """List server-side jobs across the database's tables.
+ """List server-side jobs, optionally filtered before pagination.
- Returns a lazy iterator. ``page_limit`` limits each request, not the total;
- ``page_token`` resumes from a saved token. Requests and errors occur during
- iteration. See [pagination state][lancedb.listing.AsyncListing].
+ ``table`` names a table in ``namespace_path`` (the root by default).
+ Listings include all branches of the table. A namespace filter requires
+ ``table``.
+ ``state`` uses registry states such as ``in_progress``, ``succeeded``,
+ ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
+ ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
+ Creation-time bounds are Unix milliseconds, inclusive at
+ ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
+
+ Returns a lazy iterator. ``page_limit`` limits each request, not the total.
+ Resume a ``page_token`` with the same filters. Requests and errors occur
+ during iteration. See [pagination state][lancedb.listing.AsyncListing].
+
+ For example, to inspect up to five jobs without fetching all history::
+
+ jobs = db.list_jobs(table="events", page_limit=5)
+ first_five = jobs[:5]
"""
return Listing(
- self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
+ self._conn.list_jobs(
+ page_token=page_token,
+ page_limit=page_limit,
+ table=table,
+ namespace_path=namespace_path,
+ state=state,
+ job_type=job_type,
+ job_types=job_types,
+ created_at_gte_millis=created_at_gte_millis,
+ created_at_lt_millis=created_at_lt_millis,
+ )
)
@override
diff --git a/python/python/tests/test_listing.py b/python/python/tests/test_listing.py
index cc1c44e90..d18e8580b 100644
--- a/python/python/tests/test_listing.py
+++ b/python/python/tests/test_listing.py
@@ -268,7 +268,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous)
if kind == "jobs":
assert verb == "POST"
assert path.path == "/v1/jobs/list"
- assert body == {"page_token": token, "limit": 2}
+ assert body == {"database": "dev", "page_token": token, "limit": 2}
else:
namespace = "team"
resource = kind.removesuffix("s")
@@ -326,3 +326,179 @@ async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_l
with pytest.raises(ValueError, match="limit"):
await items.__anext__()
assert requests == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("namespace", [[], ["team", "project"]])
+@pytest.mark.parametrize("multiple_types", [False, True])
+async def test_job_filters_persist_across_pages_and_resume(
+ listing_server, asynchronous, namespace, multiple_types
+):
+ endpoint, requests, responses = listing_server
+ responses.extend(
+ (200, response_page("jobs", names, token))
+ for names, token in [(["a"], "empty"), ([], "last"), (["b"], None)]
+ )
+ db = await connection(endpoint, asynchronous)
+ types = ["column_refresh", "materialized_view_refresh"]
+ filters = dict(
+ table="events",
+ namespace_path=namespace,
+ state="in_progress",
+ created_at_gte_millis=100,
+ created_at_lt_millis=200,
+ **({"job_types": types} if multiple_types else {"job_type": types[0]}),
+ )
+ items = db.list_jobs(page_limit=1, **filters)
+ assert requests == []
+ assert (await advance(items, asynchronous)).job_id == "a"
+ assert len(requests) == 1
+ resumed = db.list_jobs(page_token=items.page_token(), page_limit=1, **filters)
+ assert (await advance(resumed, asynchronous)).job_id == "b"
+ assert resumed.page_token() is None
+ assert len(requests) == 3
+ expected = dict(
+ database="dev",
+ limit=1,
+ table_ref={
+ "db_name": "dev",
+ "table_id": {"namespace_path": namespace, "table_name": "events"},
+ },
+ state="in_progress",
+ job_types=types if multiple_types else types[:1],
+ created_at_gte_millis=100,
+ created_at_lt_millis=200,
+ )
+ for (_, path, body), token in zip(requests, [None, "empty", "last"]):
+ assert path == "/v1/jobs/list"
+ assert body == {**expected, **({"page_token": token} if token else {})}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize(
+ "filters",
+ [
+ {},
+ {"state": "failed"},
+ {"job_types": ["column_refresh"]},
+ {"created_at_lt_millis": 200},
+ ],
+)
+async def test_job_listing_without_table_stays_in_connected_database(
+ listing_server, asynchronous, filters
+):
+ endpoint, requests, responses = listing_server
+ responses.append((200, response_page("jobs", ["a"], None)))
+ db = await connection(endpoint, asynchronous)
+ items = db.list_jobs(**filters)
+ assert (await advance(items, asynchronous)).job_id == "a"
+ assert requests[0][2] == {"database": "dev", **filters}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize(
+ "filters, message",
+ [
+ ({"namespace_path": ["team"]}, "requires a table"),
+ ({"created_at_gte_millis": 200, "created_at_lt_millis": 200}, "less than"),
+ ({"created_at_gte_millis": 201, "created_at_lt_millis": 200}, "less than"),
+ ({"job_type": "column_refresh", "job_types": []}, "only one"),
+ ({"state": "unknown"}, "state must be"),
+ ],
+)
+async def test_invalid_job_filters_do_not_fetch(
+ listing_server, asynchronous, filters, message
+):
+ endpoint, requests, _ = listing_server
+ db = await connection(endpoint, asynchronous)
+ with pytest.raises(ValueError, match=message):
+ items = db.list_jobs(**filters)
+ await advance(items, asynchronous)
+ assert requests == []
+
+
+@pytest.mark.asyncio
+async def test_listing_slice_and_index_fetch_only_needed_pages(listing_server):
+ endpoint, requests, responses = listing_server
+ responses.extend(
+ (200, response_page("jobs", names, token))
+ for names, token in [(["a", "b"], "second"), (["c", "d"], "last")]
+ )
+ db = await connection(endpoint, False)
+ items = db.list_jobs(table="events", page_limit=2)
+ assert items[:0] == []
+ assert items[5:2] == []
+ assert requests == []
+ assert [job.job_id for job in items[:2]] == ["a", "b"]
+ assert len(requests) == 1
+ assert items.num_page_results() == 2
+ assert items.page_token() == "second"
+ assert items[1].job_id == "b"
+ assert next(items).job_id == "a"
+ assert [job.job_id for job in items[:2]] == ["b", "c"]
+ assert len(requests) == 2
+ assert items.num_page_results() == 3
+ assert items.page_token() == "last"
+ assert [job.job_id for job in items[0:3:2][:1]] == ["b"]
+ assert len(requests) == 2
+
+
+@pytest.mark.asyncio
+async def test_listing_length_and_slices_preserve_remaining_items(listing_server):
+ endpoint, requests, responses = listing_server
+ responses.extend(
+ (200, response_page("jobs", names, token))
+ for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], None)]
+ )
+ db = await connection(endpoint, False)
+ items = db.list_jobs(page_limit=2)
+ assert bool(items)
+ assert len(requests) == 1
+ assert next(items).job_id == "a"
+ assert len(items) == 3
+ assert len(requests) == 3
+ assert items.num_page_results() == 3
+ assert items.page_token() is None
+ assert items[-1].job_id == "d"
+ assert [job.job_id for job in items[::-1]] == ["d", "c", "b"]
+ assert [job.job_id for job in items[1:]] == ["c", "d"]
+ assert [job.job_id for job in items[:100:2]] == ["b", "d"]
+ assert [job.job_id for job in items[: 2**100 : 2**100]] == ["b"]
+ with pytest.raises(IndexError):
+ _ = items[3]
+ assert [job.job_id for job in items] == ["b", "c", "d"]
+ assert len(items) == 0
+ assert not items
+ assert len(requests) == 3
+
+
+@pytest.mark.asyncio
+async def test_listing_invalid_slice_does_not_fetch(listing_server):
+ endpoint, requests, _ = listing_server
+ db = await connection(endpoint, False)
+ items = db.list_jobs()
+ with pytest.raises(ValueError, match="step"):
+ _ = items[::0]
+ with pytest.raises(TypeError):
+ _ = items["invalid"]
+ with pytest.raises(TypeError):
+ _ = items[:1.5]
+ assert requests == []
+
+
+@pytest.mark.asyncio
+async def test_listing_slice_failure_preserves_cache_and_retry_token(listing_server):
+ endpoint, requests, responses = listing_server
+ responses.extend([(200, response_page("jobs", ["a", "b"], "retry")), (401, {})])
+ db = await connection(endpoint, False)
+ items = db.list_jobs(page_limit=2)
+ with pytest.raises(HttpError):
+ _ = items[:3]
+ assert items.page_token() == "retry"
+ assert items.num_page_results() == 2
+ assert [job.job_id for job in items[:2]] == ["a", "b"]
+ assert [job.job_id for job in items] == ["a", "b"]
+ assert len(requests) == 2
diff --git a/python/src/connection.rs b/python/src/connection.rs
index 1a171032c..2d65a0459 100644
--- a/python/src/connection.rs
+++ b/python/src/connection.rs
@@ -23,7 +23,7 @@ use lancedb::{
connection::Connection as LanceConnection,
connection::NamespaceClientPushdownOperation,
database::namespace::LanceNamespaceDatabase,
- database::{CreateTableMode, Database, ReadConsistency},
+ database::{CreateTableMode, Database, JobListFilter, JobListState, ReadConsistency},
function::{GpuClass as LanceGpuClass, RefreshColumnResult},
graph::PropertyGraphDefinition,
materialized_view::CreateMaterializedViewBuilder,
@@ -1319,15 +1319,47 @@ impl Connection {
))
}
- #[pyo3(signature = (*, page_token=None, page_limit=None))]
+ #[allow(clippy::too_many_arguments)]
+ #[pyo3(signature = (*, page_token=None, page_limit=None, table=None, namespace_path=None, state=None, job_type=None, job_types=None, created_at_gte_millis=None, created_at_lt_millis=None))]
pub fn list_jobs(
&self,
page_token: Option,
page_limit: Option,
+ table: Option,
+ namespace_path: Option>,
+ state: Option,
+ job_type: Option,
+ job_types: Option>,
+ created_at_gte_millis: Option,
+ created_at_lt_millis: Option,
) -> PyResult {
+ if job_type.is_some() && job_types.is_some() {
+ return Err(PyValueError::new_err(
+ "set only one of job_type and job_types",
+ ));
+ }
+ let mut filter = JobListFilter::default();
+ filter.table = table;
+ filter.namespace_path = namespace_path.unwrap_or_default();
+ filter.state = state
+ .as_deref()
+ .map(|state| match state {
+ "in_progress" => Ok(JobListState::InProgress),
+ "succeeded" => Ok(JobListState::Succeeded),
+ "failed" => Ok(JobListState::Failed),
+ "canceled" => Ok(JobListState::Canceled),
+ "timed_out" => Ok(JobListState::TimedOut),
+ _ => Err(PyValueError::new_err(
+ "state must be one of: in_progress, succeeded, failed, canceled, timed_out",
+ )),
+ })
+ .transpose()?;
+ filter.job_types = job_types.unwrap_or_else(|| job_type.into_iter().collect());
+ filter.created_at_gte_millis = created_at_gte_millis;
+ filter.created_at_lt_millis = created_at_lt_millis;
Ok(crate::listing::JobListing::new(
self.get_inner()?
- .list_jobs(crate::listing::options(page_token, page_limit)),
+ .list_jobs_filtered(filter, crate::listing::options(page_token, page_limit)),
))
}
diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs
index 6dacbbbea..29f70d164 100644
--- a/rust/lancedb/src/connection.rs
+++ b/rust/lancedb/src/connection.rs
@@ -23,8 +23,8 @@ use crate::connection::create_table::CreateTableBuilder;
use crate::data::scannable::Scannable;
use crate::database::listing::ListingDatabase;
use crate::database::{
- CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, OpenTableRequest,
- PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
+ CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, JobListFilter,
+ OpenTableRequest, PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
};
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
use crate::error::{Error, Result};
@@ -1366,6 +1366,37 @@ impl Connection {
})
}
+ /// List jobs using server-side filters, without scanning all history in the client.
+ ///
+ /// Filters are preserved on every page, including when resuming with a saved
+ /// token. Requests and errors occur during iteration, as with [`Self::list_jobs`].
+ ///
+ /// ```
+ /// # async fn example(connection: &lancedb::Connection) -> lancedb::Result<()> {
+ /// use futures::TryStreamExt;
+ /// use lancedb::{database::JobListFilter, listing::ListingOptions};
+ /// let mut filter = JobListFilter::default();
+ /// filter.table = Some("events".into());
+ /// let mut jobs = connection.list_jobs_filtered(filter, ListingOptions::default().page_limit(5));
+ /// if let Some(job) = jobs.try_next().await? {
+ /// println!("{}", job.job_id);
+ /// }
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub fn list_jobs_filtered(
+ &self,
+ filter: JobListFilter,
+ options: ListingOptions,
+ ) -> Listing {
+ let database = self.database().clone();
+ Listing::new(options, move |options| {
+ let database = database.clone();
+ let filter = filter.clone();
+ async move { database.list_jobs_filtered(filter, options).await }
+ })
+ }
+
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
pub async fn cancel_job(&self, job_id: impl AsRef) -> Result {
diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs
index 638ae0dd5..8a7a941f6 100644
--- a/rust/lancedb/src/database.rs
+++ b/rust/lancedb/src/database.rs
@@ -238,6 +238,45 @@ pub enum ReadConsistency {
Strong,
}
+/// Registry lifecycle states accepted when filtering jobs.
+#[derive(Clone, Copy, Debug, serde::Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum JobListState {
+ /// A job that has not reached a terminal state.
+ InProgress,
+ /// A job that completed successfully.
+ Succeeded,
+ /// A job that failed.
+ Failed,
+ /// A job that was canceled.
+ Canceled,
+ /// A job that exceeded its timeout.
+ TimedOut,
+}
+
+/// Server-side filters applied before job pagination.
+///
+/// Continuation tokens must be resumed with the same filters. Creation times
+/// are Unix timestamps in milliseconds; the lower bound is inclusive and the
+/// upper bound is exclusive.
+#[derive(Clone, Debug, Default)]
+#[non_exhaustive]
+pub struct JobListFilter {
+ /// Table name within `namespace_path`, including all its branches; absent means all tables.
+ pub table: Option,
+ /// Namespace of `table`; requires a table name.
+ pub namespace_path: Vec,
+ /// Registry state, such as `in_progress`, `succeeded`, or `failed`.
+ /// These are server states, rather than the labels in [`JobInfo::state`].
+ pub state: Option,
+ /// Include jobs of any of these types; empty means all types.
+ pub job_types: Vec,
+ /// Inclusive creation-time lower bound.
+ pub created_at_gte_millis: Option,
+ /// Exclusive creation-time upper bound.
+ pub created_at_lt_millis: Option,
+}
+
/// A row from [`Database::list_jobs`]: one server-side job (index build,
/// compaction, column refresh, ...).
#[derive(Debug, Clone)]
@@ -781,6 +820,14 @@ pub trait Database:
async fn list_jobs(&self, _options: ListingOptions) -> Result> {
job_op_not_supported("list_jobs")
}
+ /// Fetch one page of jobs matching server-side filters.
+ async fn list_jobs_filtered(
+ &self,
+ _filter: JobListFilter,
+ _options: ListingOptions,
+ ) -> Result> {
+ job_op_not_supported("list_jobs_filtered")
+ }
/// Request cancellation of a job by id. Returns true if the server
/// accepted the cancellation, false if no such job exists. Cancelling an
/// already-terminal job is a no-op success.
diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs
index 1a825bebc..125930cb7 100644
--- a/rust/lancedb/src/remote/db.rs
+++ b/rust/lancedb/src/remote/db.rs
@@ -22,7 +22,8 @@ use lance_namespace::models::{
use crate::Error;
use crate::database::{
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions,
- ExecuteQueryRequest, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
+ ExecuteQueryRequest, JobInfo, JobListFilter, OpenTableRequest, ReadConsistency,
+ TableNamesRequest,
};
use crate::error::Result;
use crate::function::{FunctionRegistrationRequest, FunctionVersion, GpuClass};
@@ -1290,7 +1291,54 @@ impl Database for RemoteDatabase {
}
async fn list_jobs(&self, options: ListingOptions) -> Result> {
- let mut body = serde_json::json!({});
+ self.list_jobs_filtered(JobListFilter::default(), options)
+ .await
+ }
+
+ async fn list_jobs_filtered(
+ &self,
+ filter: JobListFilter,
+ options: ListingOptions,
+ ) -> Result> {
+ if self.db_name.is_empty() {
+ return Err(Error::InvalidInput {
+ message: "list_jobs requires a connected database".into(),
+ });
+ }
+ if filter.table.is_none() && !filter.namespace_path.is_empty() {
+ return Err(Error::InvalidInput {
+ message: "namespace_path requires a table filter".into(),
+ });
+ }
+ if let (Some(start), Some(end)) =
+ (filter.created_at_gte_millis, filter.created_at_lt_millis)
+ && start >= end
+ {
+ return Err(Error::InvalidInput {
+ message: "created_at_gte_millis must be less than created_at_lt_millis".into(),
+ });
+ }
+ let mut body = serde_json::json!({"database": self.db_name});
+ if let Some(table) = filter.table {
+ validate_table_name(&table)?;
+ validate_namespace(&filter.namespace_path)?;
+ body["table_ref"] = serde_json::json!({
+ "db_name": self.db_name,
+ "table_id": {"namespace_path": filter.namespace_path, "table_name": table},
+ });
+ }
+ if let Some(state) = filter.state {
+ body["state"] = serde_json::json!(state);
+ }
+ if !filter.job_types.is_empty() {
+ body["job_types"] = serde_json::json!(filter.job_types);
+ }
+ if let Some(start) = filter.created_at_gte_millis {
+ body["created_at_gte_millis"] = start.into();
+ }
+ if let Some(end) = filter.created_at_lt_millis {
+ body["created_at_lt_millis"] = end.into();
+ }
if let Some(token) = options.page_token {
body["page_token"] = token.into();
}
@@ -1999,7 +2047,7 @@ mod tests {
use rstest::rstest;
use crate::connection::ConnectBuilder;
- use crate::database::Database;
+ use crate::database::{Database, JobListFilter, JobListState};
use crate::error::{JobFailure, RemoteErrorMetadata, RemoteResourceIdentity};
use crate::graph::{EdgeTable, Endpoint, NodeTable, PropertyGraphDefinition};
use crate::materialized_view::CreateMaterializedViewRequest;
@@ -4133,7 +4181,10 @@ mod tests {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
.unwrap();
- assert_eq!(body, serde_json::json!({"page_token": token, "limit": 2}));
+ assert_eq!(
+ body,
+ serde_json::json!({"database": "default", "page_token": token, "limit": 2})
+ );
} else {
let ns = "team";
assert_eq!(
@@ -4336,6 +4387,84 @@ mod tests {
));
}
+ #[tokio::test]
+ async fn test_list_jobs_filters_every_page_without_eager_fetching() {
+ let count = Arc::new(AtomicUsize::new(0));
+ let seen = count.clone();
+ let conn = Connection::new_with_handler(move |request| {
+ let page = seen.fetch_add(1, Ordering::SeqCst);
+ let body: serde_json::Value =
+ serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
+ let mut expected = serde_json::json!({
+ "database": "default",
+ "table_ref": {
+ "db_name": "default",
+ "table_id": {"namespace_path": ["team", "project"], "table_name": "events"},
+ },
+ "state": "in_progress",
+ "job_types": ["column_refresh", "materialized_view_refresh"],
+ "created_at_gte_millis": 100,
+ "created_at_lt_millis": 200,
+ "limit": 1,
+ });
+ if page > 0 {
+ expected["page_token"] = "next".into();
+ }
+ assert_eq!(body, expected);
+ http::Response::builder().status(200).body(serde_json::json!({
+ "jobs": [{"job_id": format!("job-{page}"), "table": "events", "job_type": "column_refresh", "state": "in_progress", "created_at_millis": 100}],
+ "page_token": (page == 0).then_some("next"),
+ }).to_string()).unwrap()
+ });
+ let filter = JobListFilter {
+ table: Some("events".into()),
+ namespace_path: vec!["team".into(), "project".into()],
+ state: Some(JobListState::InProgress),
+ job_types: vec!["column_refresh".into(), "materialized_view_refresh".into()],
+ created_at_gte_millis: Some(100),
+ created_at_lt_millis: Some(200),
+ };
+ let mut jobs =
+ conn.list_jobs_filtered(filter.clone(), ListingOptions::default().page_limit(1));
+ assert_eq!(count.load(Ordering::SeqCst), 0);
+ assert_eq!(jobs.try_next().await.unwrap().unwrap().job_id, "job-0");
+ assert_eq!(count.load(Ordering::SeqCst), 1);
+ let resumed = conn.list_jobs_filtered(
+ filter,
+ ListingOptions::default()
+ .page_limit(1)
+ .page_token(jobs.page_token().unwrap()),
+ );
+ let remaining = resumed.try_collect::>().await.unwrap();
+ assert_eq!(remaining.len(), 1);
+ assert_eq!(remaining[0].job_id, "job-1");
+ assert_eq!(count.load(Ordering::SeqCst), 2);
+ }
+
+ #[tokio::test]
+ async fn test_list_jobs_invalid_filters_never_reach_server() {
+ let conn = Connection::new_with_handler(|_| -> http::Response {
+ panic!("invalid filters must not reach the server")
+ });
+ let namespace_only = JobListFilter {
+ namespace_path: vec!["team".into()],
+ ..Default::default()
+ };
+ let invalid_time = JobListFilter {
+ created_at_gte_millis: Some(200),
+ created_at_lt_millis: Some(100),
+ ..Default::default()
+ };
+ for filter in [namespace_only, invalid_time] {
+ let mut jobs = conn.list_jobs_filtered(filter, ListingOptions::default());
+ assert!(matches!(
+ jobs.try_next().await,
+ Err(Error::InvalidInput { .. })
+ ));
+ assert!(jobs.try_next().await.unwrap().is_none());
+ }
+ }
+
#[tokio::test]
async fn test_list_jobs_does_not_truncate_after_100_pages() {
let conn = Connection::new_with_handler(|request| {