From f17a07a2cda583af25f67797192ea776b8b8efa2 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 15 Nov 2026 00:50:58 -0800 Subject: [PATCH] feat: expose server-side job listing filters (#5041) Expose server-side job filters and lazy Listing indexing, slicing, and length helpers. Document cache, cursor, and SDK/server compatibility behavior. --- docs/web/basics/jobs.mdx | 51 ++++++-- nodejs/__test__/remote.test.ts | 53 +++++++-- python/python/lancedb/_lancedb.pyi | 14 ++- python/python/lancedb/db.py | 136 ++++++++++++++++++--- python/python/lancedb/listing.py | 72 ++++++++++- python/python/lancedb/remote/db.py | 48 +++++++- python/python/tests/test_listing.py | 178 +++++++++++++++++++++++++++- python/src/connection.rs | 38 +++++- rust/lancedb/src/connection.rs | 35 +++++- rust/lancedb/src/database.rs | 47 ++++++++ rust/lancedb/src/remote/db.rs | 137 ++++++++++++++++++++- 11 files changed, 756 insertions(+), 53 deletions(-) diff --git a/docs/web/basics/jobs.mdx b/docs/web/basics/jobs.mdx index 264acd728..db5b49ab5 100644 --- a/docs/web/basics/jobs.mdx +++ b/docs/web/basics/jobs.mdx @@ -177,14 +177,51 @@ for info in db.list_jobs(page_limit=50): live `Job` handles. Open an ID to inspect its full record. `page_limit` bounds each server request, not the total yielded items, and `page_token` resumes a listing. It is a lazy iterator: iteration performs requests and can raise -request errors. To materialize it, use `list(db.list_jobs(...))`; it does not -support list indexing, `len()`, or a `.to_list()` method. +request errors. Use `list(db.list_jobs(...))` to collect all remaining results; +there is no `.to_list()` method. -`Listing.num_page_results()` reports cached items remaining in the current -page and `Listing.page_token()` reports the next request's token. Drain the -cached items before saving that token, or resuming can skip them. A `None` -token after the final page can coexist with unread cached items. A resumed -listing is not a transactionally frozen view of jobs submitted in the meantime. + +Job filters and synchronous `Listing` indexing, slicing, and `len()` require +a Python SDK release that includes these APIs. Older SDKs expose only +`page_token` and `page_limit`; collect results with `list(...)` before indexing +or counting them. Filtering also requires a matching Enterprise server that +supports the requested filters. + + +Filter jobs on the server before pagination with `table`, `state`, `job_type` +or `job_types`, and creation-time bounds. `table` names a table in +`namespace_path` (the root by default) and includes all its branches. Requests +stay scoped to the connected database. Creation times are Unix milliseconds: +`created_at_gte_millis` is inclusive and `created_at_lt_millis` is exclusive. +Choose either `job_type` or `job_types`. + +The `state` filter uses registry states: `in_progress`, `succeeded`, `failed`, +`canceled`, and `timed_out`. These differ from the SDK lifecycle labels such +as `running` and `finished` returned in `JobInfo.state`. + +```python +jobs = db.list_jobs(table="events", state="in_progress", page_limit=5) +first_five = jobs[:5] +for info in first_five: + print(info.job_id, info.state) +``` + +Synchronous indexing and slicing refer to the remaining results without +consuming them. A bounded forward slice such as `jobs[:5]` fetches only enough +pages to reach its stop. `len(jobs)` returns the remaining count by fetching +and caching every remaining result. Negative indexes and open-ended or reverse +slices also fetch and cache all remaining results. These operations can raise +request errors; later iteration drains their cache. `AsyncListing` remains an +async iterator: use `async for` to read it. + +`Listing.num_page_results()` reports cached items available without another +request, including items retained by indexing, slicing, or counting. +`Listing.page_token()` reports the next request's token. Drain all cached +items before saving that token, or resuming can skip them. Resume with the same +filters, for example `db.list_jobs(table="events", state="in_progress", +page_token=saved_token, page_limit=5)`. A `None` token after the final page can +coexist with unread cached items. A resumed listing is not a transactionally +frozen view of jobs submitted in the meantime. `events.schema` describes the Arrow response. Use `events.to_pylist()` for Python row dictionaries. Common event columns are `event_seq` (int64), `state` diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 215df992d..9baa51292 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -2318,10 +2318,30 @@ describe("remote connection jobs surface", () => { queue_reason: "Waiting for Function worker capacity", }); const descriptions = [ - `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 3000, unknownFleet)}}`, - `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 2000, knownFleet)}}`, - `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(40, 60, 1000, unknownFleet)}}`, - `{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(60, 0, 0, unknownFleet)}}`, + `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress( + 20, + 80, + 3000, + unknownFleet, + )}}`, + `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress( + 20, + 80, + 2000, + knownFleet, + )}}`, + `{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress( + 40, + 60, + 1000, + unknownFleet, + )}}`, + `{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress( + 60, + 0, + 0, + unknownFleet, + )}}`, ]; let describeCount = 0; @@ -2393,12 +2413,24 @@ describe("remote connection jobs surface", () => { it("does not derive ETA across a zero-rate, epoch, or pause boundary", async () => { const now = Date.now(); const descriptions = [ - `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 5000}}}`, - `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 4000}}}`, - `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 3000}}}`, - `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 2000}}}`, - `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1500}}}`, - `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1000}}}`, + `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${ + now - 5000 + }}}`, + `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${ + now - 4000 + }}}`, + `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${ + now - 3000 + }}}`, + `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${ + now - 2000 + }}}`, + `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${ + now - 1500 + }}}`, + `{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${ + now - 1000 + }}}`, ]; await withMockDatabase( @@ -2574,6 +2606,7 @@ describe("lazy resource listings", () => { if (kind === "jobs") { expect(url.pathname).toBe("/v1/jobs/list"); expect(request.body).toEqual({ + database: "dev", // biome-ignore lint/style/useNamingConvention: server wire format page_token: token, limit: 2, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index a86bae9d5..0cce2085e 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -300,7 +300,19 @@ class Connection(object): page_limit: Optional[int] = None, ) -> NameListing: ... def list_jobs( - self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None + self, + *, + page_token: Optional[str] = None, + page_limit: Optional[int] = None, + table: Optional[str] = None, + namespace_path: Optional[List[str]] = None, + state: Optional[ + Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"] + ] = None, + job_type: Optional[str] = None, + job_types: Optional[List[str]] = None, + created_at_gte_millis: Optional[int] = None, + created_at_lt_millis: Optional[int] = None, ) -> JobListing: ... async def cancel_job(self, job_id: str) -> bool: ... async def pause_job(self, job_id: str) -> str: ... diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 12c7edd5c..b225a6e65 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -1602,13 +1602,39 @@ class DBConnection(EnforceOverrides): ) def list_jobs( - self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None + self, + *, + page_token: Optional[str] = None, + page_limit: Optional[int] = None, + table: Optional[str] = None, + namespace_path: Optional[List[str]] = None, + state: Optional[ + Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"] + ] = None, + job_type: Optional[str] = None, + job_types: Optional[List[str]] = None, + created_at_gte_millis: Optional[int] = None, + created_at_lt_millis: Optional[int] = None, ) -> Listing[JobInfo]: - """List server-side jobs across the database's tables. + """List server-side jobs, optionally filtered before pagination. - Returns a lazy iterator. ``page_limit`` limits each request, not the total; - ``page_token`` resumes from a saved token. Requests and errors occur during - iteration. See [pagination state][lancedb.listing.AsyncListing]. + ``table`` names a table in ``namespace_path`` (the root by default). + Listings include all branches of the table. A namespace filter requires + ``table``. + ``state`` uses registry states such as ``in_progress``, ``succeeded``, + ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's + ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``. + Creation-time bounds are Unix milliseconds, inclusive at + ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``. + + Returns a lazy iterator. ``page_limit`` limits each request, not the total. + Resume a ``page_token`` with the same filters. Requests and errors occur + during iteration. See [pagination state][lancedb.listing.AsyncListing]. + + For example, to inspect up to five jobs without fetching all history:: + + jobs = db.list_jobs(table="events", page_limit=5) + first_five = jobs[:5] """ raise NotImplementedError( _append_doc_url( @@ -2799,16 +2825,52 @@ class LanceDBConnection(DBConnection): @override def list_jobs( - self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None + self, + *, + page_token: Optional[str] = None, + page_limit: Optional[int] = None, + table: Optional[str] = None, + namespace_path: Optional[List[str]] = None, + state: Optional[ + Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"] + ] = None, + job_type: Optional[str] = None, + job_types: Optional[List[str]] = None, + created_at_gte_millis: Optional[int] = None, + created_at_lt_millis: Optional[int] = None, ) -> Listing[JobInfo]: - """List server-side jobs across the database's tables. + """List server-side jobs, optionally filtered before pagination. - Returns a lazy iterator. ``page_limit`` limits each request, not the total; - ``page_token`` resumes from a saved token. Requests and errors occur during - iteration. See [pagination state][lancedb.listing.AsyncListing]. + ``table`` names a table in ``namespace_path`` (the root by default). + Listings include all branches of the table. A namespace filter requires + ``table``. + ``state`` uses registry states such as ``in_progress``, ``succeeded``, + ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's + ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``. + Creation-time bounds are Unix milliseconds, inclusive at + ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``. + + Returns a lazy iterator. ``page_limit`` limits each request, not the total. + Resume a ``page_token`` with the same filters. Requests and errors occur + during iteration. See [pagination state][lancedb.listing.AsyncListing]. + + For example, to inspect up to five jobs without fetching all history:: + + jobs = db.list_jobs(table="events", page_limit=5) + first_five = jobs[:5] """ return Listing( - self._conn.list_jobs(page_token=page_token, page_limit=page_limit) + self._conn.list_jobs( + page_token=page_token, + page_limit=page_limit, + table=table, + namespace_path=namespace_path, + state=state, + job_type=job_type, + job_types=job_types, + created_at_gte_millis=created_at_gte_millis, + created_at_lt_millis=created_at_lt_millis, + ) ) @override @@ -4375,16 +4437,56 @@ class AsyncConnection(object): ) def list_jobs( - self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None + self, + *, + page_token: Optional[str] = None, + page_limit: Optional[int] = None, + table: Optional[str] = None, + namespace_path: Optional[List[str]] = None, + state: Optional[ + Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"] + ] = None, + job_type: Optional[str] = None, + job_types: Optional[List[str]] = None, + created_at_gte_millis: Optional[int] = None, + created_at_lt_millis: Optional[int] = None, ) -> AsyncListing[JobInfo]: - """List server-side jobs across the database's tables. + """List server-side jobs, optionally filtered before pagination. - Returns a lazy iterator. ``page_limit`` limits each request, not the total; - ``page_token`` resumes from a saved token. Requests and errors occur during - iteration. See [pagination state][lancedb.listing.AsyncListing]. + ``table`` names a table in ``namespace_path`` (the root by default). + Listings include all branches of the table. A namespace filter requires + ``table``. + ``state`` uses registry states such as ``in_progress``, ``succeeded``, + ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's + ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``. + Creation-time bounds are Unix milliseconds, inclusive at + ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``. + + Returns a lazy iterator. ``page_limit`` limits each request, not the total. + Resume a ``page_token`` with the same filters. Requests and errors occur + during iteration. See [pagination state][lancedb.listing.AsyncListing]. + + For example, to inspect up to five jobs without fetching all history:: + + jobs = db.list_jobs(table="events", page_limit=5) + first_five = [] + async for job in jobs: + first_five.append(job) + if len(first_five) == 5: + break """ return AsyncListing( - self._inner.list_jobs(page_token=page_token, page_limit=page_limit) + self._inner.list_jobs( + page_token=page_token, + page_limit=page_limit, + table=table, + namespace_path=namespace_path, + state=state, + job_type=job_type, + job_types=job_types, + created_at_gte_millis=created_at_gte_millis, + created_at_lt_millis=created_at_lt_millis, + ) ) async def cancel_job(self, job_id: str) -> bool: diff --git a/python/python/lancedb/listing.py b/python/python/lancedb/listing.py index 59aba1431..f85599b2d 100644 --- a/python/python/lancedb/listing.py +++ b/python/python/lancedb/listing.py @@ -6,7 +6,19 @@ from __future__ import annotations import asyncio -from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar +from collections import deque +from itertools import islice +from operator import index +from typing import ( + Any, + AsyncIterator, + Callable, + Generic, + Iterator, + Optional, + TypeVar, + overload, +) from .background_loop import LOOP @@ -59,29 +71,85 @@ class Listing(Iterator[T], Generic[T]): print(name) Use ``list(connection.list_views())`` to collect all results. + + Indexes and slices refer to the remaining items and do not consume them. + A bounded forward slice, such as ``listing[:5]``, fetches only enough pages + to reach its stop. ``len(listing)``, negative indexes, and open-ended or + reverse slices fetch and cache all remaining results. Iteration then drains + that cache. These operations may perform requests and raise request errors. """ def __init__(self, inner: AsyncListing[T]): self._inner = inner + self._buffer: deque[T] = deque() def __iter__(self) -> Listing[T]: return self def __next__(self) -> T: + if self._buffer: + return self._buffer.popleft() try: return LOOP.run(self._inner.__anext__()) except StopAsyncIteration: raise StopIteration from None + def _fill(self, count: Optional[int]) -> None: + while count is None or len(self._buffer) < count: + try: + item = LOOP.run(self._inner.__anext__()) + except StopAsyncIteration: + return + self._buffer.append(item) + + def __len__(self) -> int: + """Fetch and cache every remaining item, returning the remaining count.""" + self._fill(None) + return len(self._buffer) + + def __bool__(self) -> bool: + self._fill(1) + return bool(self._buffer) + + @overload + def __getitem__(self, key: int) -> T: ... + + @overload + def __getitem__(self, key: slice) -> list[T]: ... + + def __getitem__(self, key: int | slice) -> T | list[T]: + """Read remaining items without consuming them; bounded slices stay lazy.""" + if isinstance(key, slice): + start = 0 if key.start is None else index(key.start) + stop = None if key.stop is None else index(key.stop) + step = 1 if key.step is None else index(key.step) + if step == 0: + raise ValueError("slice step cannot be zero") + if start >= 0 and stop is not None and stop >= 0 and step > 0: + if start >= stop: + return [] + self._fill(stop) + start, stop, step = key.indices(len(self._buffer)) + return list( + islice(self._buffer, start, stop, min(step, len(self._buffer) + 1)) + ) + self._fill(None) + return list(self._buffer)[key] + position = index(key) + self._fill(None if position < 0 else position + 1) + return self._buffer[position] + def num_page_results(self) -> int: """Return the number of cached items available without another request.""" - return self._inner.num_page_results() + return len(self._buffer) + self._inner.num_page_results() def page_token(self) -> Optional[str]: """Return the next request's token. See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token] for initial, cached, final-page, and error behavior. + Indexing, slicing, and counting can also advance this token; drain all + ``num_page_results()`` cached items before resuming from it. """ return self._inner.page_token() diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 913c8fc2c..2927f2726 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -1324,16 +1324,52 @@ class RemoteDBConnection(DBConnection): @override def list_jobs( - self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None + self, + *, + page_token: Optional[str] = None, + page_limit: Optional[int] = None, + table: Optional[str] = None, + namespace_path: Optional[List[str]] = None, + state: Optional[ + Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"] + ] = None, + job_type: Optional[str] = None, + job_types: Optional[List[str]] = None, + created_at_gte_millis: Optional[int] = None, + created_at_lt_millis: Optional[int] = None, ) -> Listing[JobInfo]: - """List server-side jobs across the database's tables. + """List server-side jobs, optionally filtered before pagination. - Returns a lazy iterator. ``page_limit`` limits each request, not the total; - ``page_token`` resumes from a saved token. Requests and errors occur during - iteration. See [pagination state][lancedb.listing.AsyncListing]. + ``table`` names a table in ``namespace_path`` (the root by default). + Listings include all branches of the table. A namespace filter requires + ``table``. + ``state`` uses registry states such as ``in_progress``, ``succeeded``, + ``failed``, ``timed_out``, and ``canceled``, rather than the SDK's + ``running``/``finished`` labels. Set either ``job_type`` or ``job_types``. + Creation-time bounds are Unix milliseconds, inclusive at + ``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``. + + Returns a lazy iterator. ``page_limit`` limits each request, not the total. + Resume a ``page_token`` with the same filters. Requests and errors occur + during iteration. See [pagination state][lancedb.listing.AsyncListing]. + + For example, to inspect up to five jobs without fetching all history:: + + jobs = db.list_jobs(table="events", page_limit=5) + first_five = jobs[:5] """ return Listing( - self._conn.list_jobs(page_token=page_token, page_limit=page_limit) + self._conn.list_jobs( + page_token=page_token, + page_limit=page_limit, + table=table, + namespace_path=namespace_path, + state=state, + job_type=job_type, + job_types=job_types, + created_at_gte_millis=created_at_gte_millis, + created_at_lt_millis=created_at_lt_millis, + ) ) @override diff --git a/python/python/tests/test_listing.py b/python/python/tests/test_listing.py index cc1c44e90..d18e8580b 100644 --- a/python/python/tests/test_listing.py +++ b/python/python/tests/test_listing.py @@ -268,7 +268,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous) if kind == "jobs": assert verb == "POST" assert path.path == "/v1/jobs/list" - assert body == {"page_token": token, "limit": 2} + assert body == {"database": "dev", "page_token": token, "limit": 2} else: namespace = "team" resource = kind.removesuffix("s") @@ -326,3 +326,179 @@ async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_l with pytest.raises(ValueError, match="limit"): await items.__anext__() assert requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("namespace", [[], ["team", "project"]]) +@pytest.mark.parametrize("multiple_types", [False, True]) +async def test_job_filters_persist_across_pages_and_resume( + listing_server, asynchronous, namespace, multiple_types +): + endpoint, requests, responses = listing_server + responses.extend( + (200, response_page("jobs", names, token)) + for names, token in [(["a"], "empty"), ([], "last"), (["b"], None)] + ) + db = await connection(endpoint, asynchronous) + types = ["column_refresh", "materialized_view_refresh"] + filters = dict( + table="events", + namespace_path=namespace, + state="in_progress", + created_at_gte_millis=100, + created_at_lt_millis=200, + **({"job_types": types} if multiple_types else {"job_type": types[0]}), + ) + items = db.list_jobs(page_limit=1, **filters) + assert requests == [] + assert (await advance(items, asynchronous)).job_id == "a" + assert len(requests) == 1 + resumed = db.list_jobs(page_token=items.page_token(), page_limit=1, **filters) + assert (await advance(resumed, asynchronous)).job_id == "b" + assert resumed.page_token() is None + assert len(requests) == 3 + expected = dict( + database="dev", + limit=1, + table_ref={ + "db_name": "dev", + "table_id": {"namespace_path": namespace, "table_name": "events"}, + }, + state="in_progress", + job_types=types if multiple_types else types[:1], + created_at_gte_millis=100, + created_at_lt_millis=200, + ) + for (_, path, body), token in zip(requests, [None, "empty", "last"]): + assert path == "/v1/jobs/list" + assert body == {**expected, **({"page_token": token} if token else {})} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "filters", + [ + {}, + {"state": "failed"}, + {"job_types": ["column_refresh"]}, + {"created_at_lt_millis": 200}, + ], +) +async def test_job_listing_without_table_stays_in_connected_database( + listing_server, asynchronous, filters +): + endpoint, requests, responses = listing_server + responses.append((200, response_page("jobs", ["a"], None))) + db = await connection(endpoint, asynchronous) + items = db.list_jobs(**filters) + assert (await advance(items, asynchronous)).job_id == "a" + assert requests[0][2] == {"database": "dev", **filters} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + "filters, message", + [ + ({"namespace_path": ["team"]}, "requires a table"), + ({"created_at_gte_millis": 200, "created_at_lt_millis": 200}, "less than"), + ({"created_at_gte_millis": 201, "created_at_lt_millis": 200}, "less than"), + ({"job_type": "column_refresh", "job_types": []}, "only one"), + ({"state": "unknown"}, "state must be"), + ], +) +async def test_invalid_job_filters_do_not_fetch( + listing_server, asynchronous, filters, message +): + endpoint, requests, _ = listing_server + db = await connection(endpoint, asynchronous) + with pytest.raises(ValueError, match=message): + items = db.list_jobs(**filters) + await advance(items, asynchronous) + assert requests == [] + + +@pytest.mark.asyncio +async def test_listing_slice_and_index_fetch_only_needed_pages(listing_server): + endpoint, requests, responses = listing_server + responses.extend( + (200, response_page("jobs", names, token)) + for names, token in [(["a", "b"], "second"), (["c", "d"], "last")] + ) + db = await connection(endpoint, False) + items = db.list_jobs(table="events", page_limit=2) + assert items[:0] == [] + assert items[5:2] == [] + assert requests == [] + assert [job.job_id for job in items[:2]] == ["a", "b"] + assert len(requests) == 1 + assert items.num_page_results() == 2 + assert items.page_token() == "second" + assert items[1].job_id == "b" + assert next(items).job_id == "a" + assert [job.job_id for job in items[:2]] == ["b", "c"] + assert len(requests) == 2 + assert items.num_page_results() == 3 + assert items.page_token() == "last" + assert [job.job_id for job in items[0:3:2][:1]] == ["b"] + assert len(requests) == 2 + + +@pytest.mark.asyncio +async def test_listing_length_and_slices_preserve_remaining_items(listing_server): + endpoint, requests, responses = listing_server + responses.extend( + (200, response_page("jobs", names, token)) + for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], None)] + ) + db = await connection(endpoint, False) + items = db.list_jobs(page_limit=2) + assert bool(items) + assert len(requests) == 1 + assert next(items).job_id == "a" + assert len(items) == 3 + assert len(requests) == 3 + assert items.num_page_results() == 3 + assert items.page_token() is None + assert items[-1].job_id == "d" + assert [job.job_id for job in items[::-1]] == ["d", "c", "b"] + assert [job.job_id for job in items[1:]] == ["c", "d"] + assert [job.job_id for job in items[:100:2]] == ["b", "d"] + assert [job.job_id for job in items[: 2**100 : 2**100]] == ["b"] + with pytest.raises(IndexError): + _ = items[3] + assert [job.job_id for job in items] == ["b", "c", "d"] + assert len(items) == 0 + assert not items + assert len(requests) == 3 + + +@pytest.mark.asyncio +async def test_listing_invalid_slice_does_not_fetch(listing_server): + endpoint, requests, _ = listing_server + db = await connection(endpoint, False) + items = db.list_jobs() + with pytest.raises(ValueError, match="step"): + _ = items[::0] + with pytest.raises(TypeError): + _ = items["invalid"] + with pytest.raises(TypeError): + _ = items[:1.5] + assert requests == [] + + +@pytest.mark.asyncio +async def test_listing_slice_failure_preserves_cache_and_retry_token(listing_server): + endpoint, requests, responses = listing_server + responses.extend([(200, response_page("jobs", ["a", "b"], "retry")), (401, {})]) + db = await connection(endpoint, False) + items = db.list_jobs(page_limit=2) + with pytest.raises(HttpError): + _ = items[:3] + assert items.page_token() == "retry" + assert items.num_page_results() == 2 + assert [job.job_id for job in items[:2]] == ["a", "b"] + assert [job.job_id for job in items] == ["a", "b"] + assert len(requests) == 2 diff --git a/python/src/connection.rs b/python/src/connection.rs index 1a171032c..2d65a0459 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -23,7 +23,7 @@ use lancedb::{ connection::Connection as LanceConnection, connection::NamespaceClientPushdownOperation, database::namespace::LanceNamespaceDatabase, - database::{CreateTableMode, Database, ReadConsistency}, + database::{CreateTableMode, Database, JobListFilter, JobListState, ReadConsistency}, function::{GpuClass as LanceGpuClass, RefreshColumnResult}, graph::PropertyGraphDefinition, materialized_view::CreateMaterializedViewBuilder, @@ -1319,15 +1319,47 @@ impl Connection { )) } - #[pyo3(signature = (*, page_token=None, page_limit=None))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (*, page_token=None, page_limit=None, table=None, namespace_path=None, state=None, job_type=None, job_types=None, created_at_gte_millis=None, created_at_lt_millis=None))] pub fn list_jobs( &self, page_token: Option, page_limit: Option, + table: Option, + namespace_path: Option>, + state: Option, + job_type: Option, + job_types: Option>, + created_at_gte_millis: Option, + created_at_lt_millis: Option, ) -> PyResult { + if job_type.is_some() && job_types.is_some() { + return Err(PyValueError::new_err( + "set only one of job_type and job_types", + )); + } + let mut filter = JobListFilter::default(); + filter.table = table; + filter.namespace_path = namespace_path.unwrap_or_default(); + filter.state = state + .as_deref() + .map(|state| match state { + "in_progress" => Ok(JobListState::InProgress), + "succeeded" => Ok(JobListState::Succeeded), + "failed" => Ok(JobListState::Failed), + "canceled" => Ok(JobListState::Canceled), + "timed_out" => Ok(JobListState::TimedOut), + _ => Err(PyValueError::new_err( + "state must be one of: in_progress, succeeded, failed, canceled, timed_out", + )), + }) + .transpose()?; + filter.job_types = job_types.unwrap_or_else(|| job_type.into_iter().collect()); + filter.created_at_gte_millis = created_at_gte_millis; + filter.created_at_lt_millis = created_at_lt_millis; Ok(crate::listing::JobListing::new( self.get_inner()? - .list_jobs(crate::listing::options(page_token, page_limit)), + .list_jobs_filtered(filter, crate::listing::options(page_token, page_limit)), )) } diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 6dacbbbea..29f70d164 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -23,8 +23,8 @@ use crate::connection::create_table::CreateTableBuilder; use crate::data::scannable::Scannable; use crate::database::listing::ListingDatabase; use crate::database::{ - CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, OpenTableRequest, - PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest, + CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, JobListFilter, + OpenTableRequest, PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest, }; use crate::embeddings::{EmbeddingRegistry, MemoryRegistry}; use crate::error::{Error, Result}; @@ -1366,6 +1366,37 @@ impl Connection { }) } + /// List jobs using server-side filters, without scanning all history in the client. + /// + /// Filters are preserved on every page, including when resuming with a saved + /// token. Requests and errors occur during iteration, as with [`Self::list_jobs`]. + /// + /// ``` + /// # async fn example(connection: &lancedb::Connection) -> lancedb::Result<()> { + /// use futures::TryStreamExt; + /// use lancedb::{database::JobListFilter, listing::ListingOptions}; + /// let mut filter = JobListFilter::default(); + /// filter.table = Some("events".into()); + /// let mut jobs = connection.list_jobs_filtered(filter, ListingOptions::default().page_limit(5)); + /// if let Some(job) = jobs.try_next().await? { + /// println!("{}", job.job_id); + /// } + /// # Ok(()) + /// # } + /// ``` + pub fn list_jobs_filtered( + &self, + filter: JobListFilter, + options: ListingOptions, + ) -> Listing { + let database = self.database().clone(); + Listing::new(options, move |options| { + let database = database.clone(); + let filter = filter.clone(); + async move { database.list_jobs_filtered(filter, options).await } + }) + } + /// Request cancellation of a server-side job by id. Returns true if the /// server accepted the cancellation, false if no such job exists. pub async fn cancel_job(&self, job_id: impl AsRef) -> Result { diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 638ae0dd5..8a7a941f6 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -238,6 +238,45 @@ pub enum ReadConsistency { Strong, } +/// Registry lifecycle states accepted when filtering jobs. +#[derive(Clone, Copy, Debug, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum JobListState { + /// A job that has not reached a terminal state. + InProgress, + /// A job that completed successfully. + Succeeded, + /// A job that failed. + Failed, + /// A job that was canceled. + Canceled, + /// A job that exceeded its timeout. + TimedOut, +} + +/// Server-side filters applied before job pagination. +/// +/// Continuation tokens must be resumed with the same filters. Creation times +/// are Unix timestamps in milliseconds; the lower bound is inclusive and the +/// upper bound is exclusive. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct JobListFilter { + /// Table name within `namespace_path`, including all its branches; absent means all tables. + pub table: Option, + /// Namespace of `table`; requires a table name. + pub namespace_path: Vec, + /// Registry state, such as `in_progress`, `succeeded`, or `failed`. + /// These are server states, rather than the labels in [`JobInfo::state`]. + pub state: Option, + /// Include jobs of any of these types; empty means all types. + pub job_types: Vec, + /// Inclusive creation-time lower bound. + pub created_at_gte_millis: Option, + /// Exclusive creation-time upper bound. + pub created_at_lt_millis: Option, +} + /// A row from [`Database::list_jobs`]: one server-side job (index build, /// compaction, column refresh, ...). #[derive(Debug, Clone)] @@ -781,6 +820,14 @@ pub trait Database: async fn list_jobs(&self, _options: ListingOptions) -> Result> { job_op_not_supported("list_jobs") } + /// Fetch one page of jobs matching server-side filters. + async fn list_jobs_filtered( + &self, + _filter: JobListFilter, + _options: ListingOptions, + ) -> Result> { + job_op_not_supported("list_jobs_filtered") + } /// Request cancellation of a job by id. Returns true if the server /// accepted the cancellation, false if no such job exists. Cancelling an /// already-terminal job is a no-op success. diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 1a825bebc..125930cb7 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -22,7 +22,8 @@ use lance_namespace::models::{ use crate::Error; use crate::database::{ CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions, - ExecuteQueryRequest, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest, + ExecuteQueryRequest, JobInfo, JobListFilter, OpenTableRequest, ReadConsistency, + TableNamesRequest, }; use crate::error::Result; use crate::function::{FunctionRegistrationRequest, FunctionVersion, GpuClass}; @@ -1290,7 +1291,54 @@ impl Database for RemoteDatabase { } async fn list_jobs(&self, options: ListingOptions) -> Result> { - let mut body = serde_json::json!({}); + self.list_jobs_filtered(JobListFilter::default(), options) + .await + } + + async fn list_jobs_filtered( + &self, + filter: JobListFilter, + options: ListingOptions, + ) -> Result> { + if self.db_name.is_empty() { + return Err(Error::InvalidInput { + message: "list_jobs requires a connected database".into(), + }); + } + if filter.table.is_none() && !filter.namespace_path.is_empty() { + return Err(Error::InvalidInput { + message: "namespace_path requires a table filter".into(), + }); + } + if let (Some(start), Some(end)) = + (filter.created_at_gte_millis, filter.created_at_lt_millis) + && start >= end + { + return Err(Error::InvalidInput { + message: "created_at_gte_millis must be less than created_at_lt_millis".into(), + }); + } + let mut body = serde_json::json!({"database": self.db_name}); + if let Some(table) = filter.table { + validate_table_name(&table)?; + validate_namespace(&filter.namespace_path)?; + body["table_ref"] = serde_json::json!({ + "db_name": self.db_name, + "table_id": {"namespace_path": filter.namespace_path, "table_name": table}, + }); + } + if let Some(state) = filter.state { + body["state"] = serde_json::json!(state); + } + if !filter.job_types.is_empty() { + body["job_types"] = serde_json::json!(filter.job_types); + } + if let Some(start) = filter.created_at_gte_millis { + body["created_at_gte_millis"] = start.into(); + } + if let Some(end) = filter.created_at_lt_millis { + body["created_at_lt_millis"] = end.into(); + } if let Some(token) = options.page_token { body["page_token"] = token.into(); } @@ -1999,7 +2047,7 @@ mod tests { use rstest::rstest; use crate::connection::ConnectBuilder; - use crate::database::Database; + use crate::database::{Database, JobListFilter, JobListState}; use crate::error::{JobFailure, RemoteErrorMetadata, RemoteResourceIdentity}; use crate::graph::{EdgeTable, Endpoint, NodeTable, PropertyGraphDefinition}; use crate::materialized_view::CreateMaterializedViewRequest; @@ -4133,7 +4181,10 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()) .unwrap(); - assert_eq!(body, serde_json::json!({"page_token": token, "limit": 2})); + assert_eq!( + body, + serde_json::json!({"database": "default", "page_token": token, "limit": 2}) + ); } else { let ns = "team"; assert_eq!( @@ -4336,6 +4387,84 @@ mod tests { )); } + #[tokio::test] + async fn test_list_jobs_filters_every_page_without_eager_fetching() { + let count = Arc::new(AtomicUsize::new(0)); + let seen = count.clone(); + let conn = Connection::new_with_handler(move |request| { + let page = seen.fetch_add(1, Ordering::SeqCst); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + let mut expected = serde_json::json!({ + "database": "default", + "table_ref": { + "db_name": "default", + "table_id": {"namespace_path": ["team", "project"], "table_name": "events"}, + }, + "state": "in_progress", + "job_types": ["column_refresh", "materialized_view_refresh"], + "created_at_gte_millis": 100, + "created_at_lt_millis": 200, + "limit": 1, + }); + if page > 0 { + expected["page_token"] = "next".into(); + } + assert_eq!(body, expected); + http::Response::builder().status(200).body(serde_json::json!({ + "jobs": [{"job_id": format!("job-{page}"), "table": "events", "job_type": "column_refresh", "state": "in_progress", "created_at_millis": 100}], + "page_token": (page == 0).then_some("next"), + }).to_string()).unwrap() + }); + let filter = JobListFilter { + table: Some("events".into()), + namespace_path: vec!["team".into(), "project".into()], + state: Some(JobListState::InProgress), + job_types: vec!["column_refresh".into(), "materialized_view_refresh".into()], + created_at_gte_millis: Some(100), + created_at_lt_millis: Some(200), + }; + let mut jobs = + conn.list_jobs_filtered(filter.clone(), ListingOptions::default().page_limit(1)); + assert_eq!(count.load(Ordering::SeqCst), 0); + assert_eq!(jobs.try_next().await.unwrap().unwrap().job_id, "job-0"); + assert_eq!(count.load(Ordering::SeqCst), 1); + let resumed = conn.list_jobs_filtered( + filter, + ListingOptions::default() + .page_limit(1) + .page_token(jobs.page_token().unwrap()), + ); + let remaining = resumed.try_collect::>().await.unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].job_id, "job-1"); + assert_eq!(count.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn test_list_jobs_invalid_filters_never_reach_server() { + let conn = Connection::new_with_handler(|_| -> http::Response { + panic!("invalid filters must not reach the server") + }); + let namespace_only = JobListFilter { + namespace_path: vec!["team".into()], + ..Default::default() + }; + let invalid_time = JobListFilter { + created_at_gte_millis: Some(200), + created_at_lt_millis: Some(100), + ..Default::default() + }; + for filter in [namespace_only, invalid_time] { + let mut jobs = conn.list_jobs_filtered(filter, ListingOptions::default()); + assert!(matches!( + jobs.try_next().await, + Err(Error::InvalidInput { .. }) + )); + assert!(jobs.try_next().await.unwrap().is_none()); + } + } + #[tokio::test] async fn test_list_jobs_does_not_truncate_after_100_pages() { let conn = Connection::new_with_handler(|request| {