feat: expose server-side job listing filters (#5041)

Expose server-side job filters and lazy Listing indexing, slicing, and length helpers. Document cache, cursor, and SDK/server compatibility behavior.
This commit is contained in:
Jack Ye
2026-11-15 00:50:58 -08:00
committed by GitHub
parent 530d379fc4
commit f17a07a2cd
11 changed files with 756 additions and 53 deletions
+44 -7
View File
@@ -177,14 +177,51 @@ for info in db.list_jobs(page_limit=50):
live `Job` handles. Open an ID to inspect its full record. `page_limit` bounds
each server request, not the total yielded items, and `page_token` resumes a
listing. It is a lazy iterator: iteration performs requests and can raise
request errors. To materialize it, use `list(db.list_jobs(...))`; it does not
support list indexing, `len()`, or a `.to_list()` method.
request errors. Use `list(db.list_jobs(...))` to collect all remaining results;
there is no `.to_list()` method.
`Listing.num_page_results()` reports cached items remaining in the current
page and `Listing.page_token()` reports the next request's token. Drain the
cached items before saving that token, or resuming can skip them. A `None`
token after the final page can coexist with unread cached items. A resumed
listing is not a transactionally frozen view of jobs submitted in the meantime.
<Note>
Job filters and synchronous `Listing` indexing, slicing, and `len()` require
a Python SDK release that includes these APIs. Older SDKs expose only
`page_token` and `page_limit`; collect results with `list(...)` before indexing
or counting them. Filtering also requires a matching Enterprise server that
supports the requested filters.
</Note>
Filter jobs on the server before pagination with `table`, `state`, `job_type`
or `job_types`, and creation-time bounds. `table` names a table in
`namespace_path` (the root by default) and includes all its branches. Requests
stay scoped to the connected database. Creation times are Unix milliseconds:
`created_at_gte_millis` is inclusive and `created_at_lt_millis` is exclusive.
Choose either `job_type` or `job_types`.
The `state` filter uses registry states: `in_progress`, `succeeded`, `failed`,
`canceled`, and `timed_out`. These differ from the SDK lifecycle labels such
as `running` and `finished` returned in `JobInfo.state`.
```python
jobs = db.list_jobs(table="events", state="in_progress", page_limit=5)
first_five = jobs[:5]
for info in first_five:
print(info.job_id, info.state)
```
Synchronous indexing and slicing refer to the remaining results without
consuming them. A bounded forward slice such as `jobs[:5]` fetches only enough
pages to reach its stop. `len(jobs)` returns the remaining count by fetching
and caching every remaining result. Negative indexes and open-ended or reverse
slices also fetch and cache all remaining results. These operations can raise
request errors; later iteration drains their cache. `AsyncListing` remains an
async iterator: use `async for` to read it.
`Listing.num_page_results()` reports cached items available without another
request, including items retained by indexing, slicing, or counting.
`Listing.page_token()` reports the next request's token. Drain all cached
items before saving that token, or resuming can skip them. Resume with the same
filters, for example `db.list_jobs(table="events", state="in_progress",
page_token=saved_token, page_limit=5)`. A `None` token after the final page can
coexist with unread cached items. A resumed listing is not a transactionally
frozen view of jobs submitted in the meantime.
`events.schema` describes the Arrow response. Use `events.to_pylist()` for
Python row dictionaries. Common event columns are `event_seq` (int64), `state`
+43 -10
View File
@@ -2318,10 +2318,30 @@ describe("remote connection jobs surface", () => {
queue_reason: "Waiting for Function worker capacity",
});
const descriptions = [
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 3000, unknownFleet)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 2000, knownFleet)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(40, 60, 1000, unknownFleet)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(60, 0, 0, unknownFleet)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
20,
80,
3000,
unknownFleet,
)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
20,
80,
2000,
knownFleet,
)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
40,
60,
1000,
unknownFleet,
)}}`,
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(
60,
0,
0,
unknownFleet,
)}}`,
];
let describeCount = 0;
@@ -2393,12 +2413,24 @@ describe("remote connection jobs surface", () => {
it("does not derive ETA across a zero-rate, epoch, or pause boundary", async () => {
const now = Date.now();
const descriptions = [
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 5000}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 4000}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 3000}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 2000}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1500}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1000}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${
now - 5000
}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${
now - 4000
}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
now - 3000
}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
now - 2000
}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
now - 1500
}}}`,
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
now - 1000
}}}`,
];
await withMockDatabase(
@@ -2574,6 +2606,7 @@ describe("lazy resource listings", () => {
if (kind === "jobs") {
expect(url.pathname).toBe("/v1/jobs/list");
expect(request.body).toEqual({
database: "dev",
// biome-ignore lint/style/useNamingConvention: server wire format
page_token: token,
limit: 2,
+13 -1
View File
@@ -300,7 +300,19 @@ class Connection(object):
page_limit: Optional[int] = None,
) -> NameListing: ...
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> JobListing: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def pause_job(self, job_id: str) -> str: ...
+119 -17
View File
@@ -1602,13 +1602,39 @@ class DBConnection(EnforceOverrides):
)
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = jobs[:5]
"""
raise NotImplementedError(
_append_doc_url(
@@ -2799,16 +2825,52 @@ class LanceDBConnection(DBConnection):
@override
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = jobs[:5]
"""
return Listing(
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
self._conn.list_jobs(
page_token=page_token,
page_limit=page_limit,
table=table,
namespace_path=namespace_path,
state=state,
job_type=job_type,
job_types=job_types,
created_at_gte_millis=created_at_gte_millis,
created_at_lt_millis=created_at_lt_millis,
)
)
@override
@@ -4375,16 +4437,56 @@ class AsyncConnection(object):
)
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> AsyncListing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = []
async for job in jobs:
first_five.append(job)
if len(first_five) == 5:
break
"""
return AsyncListing(
self._inner.list_jobs(page_token=page_token, page_limit=page_limit)
self._inner.list_jobs(
page_token=page_token,
page_limit=page_limit,
table=table,
namespace_path=namespace_path,
state=state,
job_type=job_type,
job_types=job_types,
created_at_gte_millis=created_at_gte_millis,
created_at_lt_millis=created_at_lt_millis,
)
)
async def cancel_job(self, job_id: str) -> bool:
+70 -2
View File
@@ -6,7 +6,19 @@
from __future__ import annotations
import asyncio
from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
from collections import deque
from itertools import islice
from operator import index
from typing import (
Any,
AsyncIterator,
Callable,
Generic,
Iterator,
Optional,
TypeVar,
overload,
)
from .background_loop import LOOP
@@ -59,29 +71,85 @@ class Listing(Iterator[T], Generic[T]):
print(name)
Use ``list(connection.list_views())`` to collect all results.
Indexes and slices refer to the remaining items and do not consume them.
A bounded forward slice, such as ``listing[:5]``, fetches only enough pages
to reach its stop. ``len(listing)``, negative indexes, and open-ended or
reverse slices fetch and cache all remaining results. Iteration then drains
that cache. These operations may perform requests and raise request errors.
"""
def __init__(self, inner: AsyncListing[T]):
self._inner = inner
self._buffer: deque[T] = deque()
def __iter__(self) -> Listing[T]:
return self
def __next__(self) -> T:
if self._buffer:
return self._buffer.popleft()
try:
return LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
raise StopIteration from None
def _fill(self, count: Optional[int]) -> None:
while count is None or len(self._buffer) < count:
try:
item = LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
return
self._buffer.append(item)
def __len__(self) -> int:
"""Fetch and cache every remaining item, returning the remaining count."""
self._fill(None)
return len(self._buffer)
def __bool__(self) -> bool:
self._fill(1)
return bool(self._buffer)
@overload
def __getitem__(self, key: int) -> T: ...
@overload
def __getitem__(self, key: slice) -> list[T]: ...
def __getitem__(self, key: int | slice) -> T | list[T]:
"""Read remaining items without consuming them; bounded slices stay lazy."""
if isinstance(key, slice):
start = 0 if key.start is None else index(key.start)
stop = None if key.stop is None else index(key.stop)
step = 1 if key.step is None else index(key.step)
if step == 0:
raise ValueError("slice step cannot be zero")
if start >= 0 and stop is not None and stop >= 0 and step > 0:
if start >= stop:
return []
self._fill(stop)
start, stop, step = key.indices(len(self._buffer))
return list(
islice(self._buffer, start, stop, min(step, len(self._buffer) + 1))
)
self._fill(None)
return list(self._buffer)[key]
position = index(key)
self._fill(None if position < 0 else position + 1)
return self._buffer[position]
def num_page_results(self) -> int:
"""Return the number of cached items available without another request."""
return self._inner.num_page_results()
return len(self._buffer) + self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the next request's token.
See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token]
for initial, cached, final-page, and error behavior.
Indexing, slicing, and counting can also advance this token; drain all
``num_page_results()`` cached items before resuming from it.
"""
return self._inner.page_token()
+42 -6
View File
@@ -1324,16 +1324,52 @@ class RemoteDBConnection(DBConnection):
@override
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = jobs[:5]
"""
return Listing(
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
self._conn.list_jobs(
page_token=page_token,
page_limit=page_limit,
table=table,
namespace_path=namespace_path,
state=state,
job_type=job_type,
job_types=job_types,
created_at_gte_millis=created_at_gte_millis,
created_at_lt_millis=created_at_lt_millis,
)
)
@override
+177 -1
View File
@@ -268,7 +268,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous)
if kind == "jobs":
assert verb == "POST"
assert path.path == "/v1/jobs/list"
assert body == {"page_token": token, "limit": 2}
assert body == {"database": "dev", "page_token": token, "limit": 2}
else:
namespace = "team"
resource = kind.removesuffix("s")
@@ -326,3 +326,179 @@ async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_l
with pytest.raises(ValueError, match="limit"):
await items.__anext__()
assert requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("namespace", [[], ["team", "project"]])
@pytest.mark.parametrize("multiple_types", [False, True])
async def test_job_filters_persist_across_pages_and_resume(
listing_server, asynchronous, namespace, multiple_types
):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page("jobs", names, token))
for names, token in [(["a"], "empty"), ([], "last"), (["b"], None)]
)
db = await connection(endpoint, asynchronous)
types = ["column_refresh", "materialized_view_refresh"]
filters = dict(
table="events",
namespace_path=namespace,
state="in_progress",
created_at_gte_millis=100,
created_at_lt_millis=200,
**({"job_types": types} if multiple_types else {"job_type": types[0]}),
)
items = db.list_jobs(page_limit=1, **filters)
assert requests == []
assert (await advance(items, asynchronous)).job_id == "a"
assert len(requests) == 1
resumed = db.list_jobs(page_token=items.page_token(), page_limit=1, **filters)
assert (await advance(resumed, asynchronous)).job_id == "b"
assert resumed.page_token() is None
assert len(requests) == 3
expected = dict(
database="dev",
limit=1,
table_ref={
"db_name": "dev",
"table_id": {"namespace_path": namespace, "table_name": "events"},
},
state="in_progress",
job_types=types if multiple_types else types[:1],
created_at_gte_millis=100,
created_at_lt_millis=200,
)
for (_, path, body), token in zip(requests, [None, "empty", "last"]):
assert path == "/v1/jobs/list"
assert body == {**expected, **({"page_token": token} if token else {})}
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
"filters",
[
{},
{"state": "failed"},
{"job_types": ["column_refresh"]},
{"created_at_lt_millis": 200},
],
)
async def test_job_listing_without_table_stays_in_connected_database(
listing_server, asynchronous, filters
):
endpoint, requests, responses = listing_server
responses.append((200, response_page("jobs", ["a"], None)))
db = await connection(endpoint, asynchronous)
items = db.list_jobs(**filters)
assert (await advance(items, asynchronous)).job_id == "a"
assert requests[0][2] == {"database": "dev", **filters}
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
"filters, message",
[
({"namespace_path": ["team"]}, "requires a table"),
({"created_at_gte_millis": 200, "created_at_lt_millis": 200}, "less than"),
({"created_at_gte_millis": 201, "created_at_lt_millis": 200}, "less than"),
({"job_type": "column_refresh", "job_types": []}, "only one"),
({"state": "unknown"}, "state must be"),
],
)
async def test_invalid_job_filters_do_not_fetch(
listing_server, asynchronous, filters, message
):
endpoint, requests, _ = listing_server
db = await connection(endpoint, asynchronous)
with pytest.raises(ValueError, match=message):
items = db.list_jobs(**filters)
await advance(items, asynchronous)
assert requests == []
@pytest.mark.asyncio
async def test_listing_slice_and_index_fetch_only_needed_pages(listing_server):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page("jobs", names, token))
for names, token in [(["a", "b"], "second"), (["c", "d"], "last")]
)
db = await connection(endpoint, False)
items = db.list_jobs(table="events", page_limit=2)
assert items[:0] == []
assert items[5:2] == []
assert requests == []
assert [job.job_id for job in items[:2]] == ["a", "b"]
assert len(requests) == 1
assert items.num_page_results() == 2
assert items.page_token() == "second"
assert items[1].job_id == "b"
assert next(items).job_id == "a"
assert [job.job_id for job in items[:2]] == ["b", "c"]
assert len(requests) == 2
assert items.num_page_results() == 3
assert items.page_token() == "last"
assert [job.job_id for job in items[0:3:2][:1]] == ["b"]
assert len(requests) == 2
@pytest.mark.asyncio
async def test_listing_length_and_slices_preserve_remaining_items(listing_server):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page("jobs", names, token))
for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], None)]
)
db = await connection(endpoint, False)
items = db.list_jobs(page_limit=2)
assert bool(items)
assert len(requests) == 1
assert next(items).job_id == "a"
assert len(items) == 3
assert len(requests) == 3
assert items.num_page_results() == 3
assert items.page_token() is None
assert items[-1].job_id == "d"
assert [job.job_id for job in items[::-1]] == ["d", "c", "b"]
assert [job.job_id for job in items[1:]] == ["c", "d"]
assert [job.job_id for job in items[:100:2]] == ["b", "d"]
assert [job.job_id for job in items[: 2**100 : 2**100]] == ["b"]
with pytest.raises(IndexError):
_ = items[3]
assert [job.job_id for job in items] == ["b", "c", "d"]
assert len(items) == 0
assert not items
assert len(requests) == 3
@pytest.mark.asyncio
async def test_listing_invalid_slice_does_not_fetch(listing_server):
endpoint, requests, _ = listing_server
db = await connection(endpoint, False)
items = db.list_jobs()
with pytest.raises(ValueError, match="step"):
_ = items[::0]
with pytest.raises(TypeError):
_ = items["invalid"]
with pytest.raises(TypeError):
_ = items[:1.5]
assert requests == []
@pytest.mark.asyncio
async def test_listing_slice_failure_preserves_cache_and_retry_token(listing_server):
endpoint, requests, responses = listing_server
responses.extend([(200, response_page("jobs", ["a", "b"], "retry")), (401, {})])
db = await connection(endpoint, False)
items = db.list_jobs(page_limit=2)
with pytest.raises(HttpError):
_ = items[:3]
assert items.page_token() == "retry"
assert items.num_page_results() == 2
assert [job.job_id for job in items[:2]] == ["a", "b"]
assert [job.job_id for job in items] == ["a", "b"]
assert len(requests) == 2
+35 -3
View File
@@ -23,7 +23,7 @@ use lancedb::{
connection::Connection as LanceConnection,
connection::NamespaceClientPushdownOperation,
database::namespace::LanceNamespaceDatabase,
database::{CreateTableMode, Database, ReadConsistency},
database::{CreateTableMode, Database, JobListFilter, JobListState, ReadConsistency},
function::{GpuClass as LanceGpuClass, RefreshColumnResult},
graph::PropertyGraphDefinition,
materialized_view::CreateMaterializedViewBuilder,
@@ -1319,15 +1319,47 @@ impl Connection {
))
}
#[pyo3(signature = (*, page_token=None, page_limit=None))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (*, page_token=None, page_limit=None, table=None, namespace_path=None, state=None, job_type=None, job_types=None, created_at_gte_millis=None, created_at_lt_millis=None))]
pub fn list_jobs(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
table: Option<String>,
namespace_path: Option<Vec<String>>,
state: Option<String>,
job_type: Option<String>,
job_types: Option<Vec<String>>,
created_at_gte_millis: Option<i64>,
created_at_lt_millis: Option<i64>,
) -> PyResult<crate::listing::JobListing> {
if job_type.is_some() && job_types.is_some() {
return Err(PyValueError::new_err(
"set only one of job_type and job_types",
));
}
let mut filter = JobListFilter::default();
filter.table = table;
filter.namespace_path = namespace_path.unwrap_or_default();
filter.state = state
.as_deref()
.map(|state| match state {
"in_progress" => Ok(JobListState::InProgress),
"succeeded" => Ok(JobListState::Succeeded),
"failed" => Ok(JobListState::Failed),
"canceled" => Ok(JobListState::Canceled),
"timed_out" => Ok(JobListState::TimedOut),
_ => Err(PyValueError::new_err(
"state must be one of: in_progress, succeeded, failed, canceled, timed_out",
)),
})
.transpose()?;
filter.job_types = job_types.unwrap_or_else(|| job_type.into_iter().collect());
filter.created_at_gte_millis = created_at_gte_millis;
filter.created_at_lt_millis = created_at_lt_millis;
Ok(crate::listing::JobListing::new(
self.get_inner()?
.list_jobs(crate::listing::options(page_token, page_limit)),
.list_jobs_filtered(filter, crate::listing::options(page_token, page_limit)),
))
}
+33 -2
View File
@@ -23,8 +23,8 @@ use crate::connection::create_table::CreateTableBuilder;
use crate::data::scannable::Scannable;
use crate::database::listing::ListingDatabase;
use crate::database::{
CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, OpenTableRequest,
PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, JobListFilter,
OpenTableRequest, PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
};
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
use crate::error::{Error, Result};
@@ -1366,6 +1366,37 @@ impl Connection {
})
}
/// List jobs using server-side filters, without scanning all history in the client.
///
/// Filters are preserved on every page, including when resuming with a saved
/// token. Requests and errors occur during iteration, as with [`Self::list_jobs`].
///
/// ```
/// # async fn example(connection: &lancedb::Connection) -> lancedb::Result<()> {
/// use futures::TryStreamExt;
/// use lancedb::{database::JobListFilter, listing::ListingOptions};
/// let mut filter = JobListFilter::default();
/// filter.table = Some("events".into());
/// let mut jobs = connection.list_jobs_filtered(filter, ListingOptions::default().page_limit(5));
/// if let Some(job) = jobs.try_next().await? {
/// println!("{}", job.job_id);
/// }
/// # Ok(())
/// # }
/// ```
pub fn list_jobs_filtered(
&self,
filter: JobListFilter,
options: ListingOptions,
) -> Listing<JobInfo> {
let database = self.database().clone();
Listing::new(options, move |options| {
let database = database.clone();
let filter = filter.clone();
async move { database.list_jobs_filtered(filter, options).await }
})
}
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
pub async fn cancel_job(&self, job_id: impl AsRef<str>) -> Result<bool> {
+47
View File
@@ -238,6 +238,45 @@ pub enum ReadConsistency {
Strong,
}
/// Registry lifecycle states accepted when filtering jobs.
#[derive(Clone, Copy, Debug, serde::Serialize)]
#[serde(rename_all = "snake_case")]
pub enum JobListState {
/// A job that has not reached a terminal state.
InProgress,
/// A job that completed successfully.
Succeeded,
/// A job that failed.
Failed,
/// A job that was canceled.
Canceled,
/// A job that exceeded its timeout.
TimedOut,
}
/// Server-side filters applied before job pagination.
///
/// Continuation tokens must be resumed with the same filters. Creation times
/// are Unix timestamps in milliseconds; the lower bound is inclusive and the
/// upper bound is exclusive.
#[derive(Clone, Debug, Default)]
#[non_exhaustive]
pub struct JobListFilter {
/// Table name within `namespace_path`, including all its branches; absent means all tables.
pub table: Option<String>,
/// Namespace of `table`; requires a table name.
pub namespace_path: Vec<String>,
/// Registry state, such as `in_progress`, `succeeded`, or `failed`.
/// These are server states, rather than the labels in [`JobInfo::state`].
pub state: Option<JobListState>,
/// Include jobs of any of these types; empty means all types.
pub job_types: Vec<String>,
/// Inclusive creation-time lower bound.
pub created_at_gte_millis: Option<i64>,
/// Exclusive creation-time upper bound.
pub created_at_lt_millis: Option<i64>,
}
/// A row from [`Database::list_jobs`]: one server-side job (index build,
/// compaction, column refresh, ...).
#[derive(Debug, Clone)]
@@ -781,6 +820,14 @@ pub trait Database:
async fn list_jobs(&self, _options: ListingOptions) -> Result<ListingPage<JobInfo>> {
job_op_not_supported("list_jobs")
}
/// Fetch one page of jobs matching server-side filters.
async fn list_jobs_filtered(
&self,
_filter: JobListFilter,
_options: ListingOptions,
) -> Result<ListingPage<JobInfo>> {
job_op_not_supported("list_jobs_filtered")
}
/// Request cancellation of a job by id. Returns true if the server
/// accepted the cancellation, false if no such job exists. Cancelling an
/// already-terminal job is a no-op success.
+133 -4
View File
@@ -22,7 +22,8 @@ use lance_namespace::models::{
use crate::Error;
use crate::database::{
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions,
ExecuteQueryRequest, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
ExecuteQueryRequest, JobInfo, JobListFilter, OpenTableRequest, ReadConsistency,
TableNamesRequest,
};
use crate::error::Result;
use crate::function::{FunctionRegistrationRequest, FunctionVersion, GpuClass};
@@ -1290,7 +1291,54 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn list_jobs(&self, options: ListingOptions) -> Result<ListingPage<JobInfo>> {
let mut body = serde_json::json!({});
self.list_jobs_filtered(JobListFilter::default(), options)
.await
}
async fn list_jobs_filtered(
&self,
filter: JobListFilter,
options: ListingOptions,
) -> Result<ListingPage<JobInfo>> {
if self.db_name.is_empty() {
return Err(Error::InvalidInput {
message: "list_jobs requires a connected database".into(),
});
}
if filter.table.is_none() && !filter.namespace_path.is_empty() {
return Err(Error::InvalidInput {
message: "namespace_path requires a table filter".into(),
});
}
if let (Some(start), Some(end)) =
(filter.created_at_gte_millis, filter.created_at_lt_millis)
&& start >= end
{
return Err(Error::InvalidInput {
message: "created_at_gte_millis must be less than created_at_lt_millis".into(),
});
}
let mut body = serde_json::json!({"database": self.db_name});
if let Some(table) = filter.table {
validate_table_name(&table)?;
validate_namespace(&filter.namespace_path)?;
body["table_ref"] = serde_json::json!({
"db_name": self.db_name,
"table_id": {"namespace_path": filter.namespace_path, "table_name": table},
});
}
if let Some(state) = filter.state {
body["state"] = serde_json::json!(state);
}
if !filter.job_types.is_empty() {
body["job_types"] = serde_json::json!(filter.job_types);
}
if let Some(start) = filter.created_at_gte_millis {
body["created_at_gte_millis"] = start.into();
}
if let Some(end) = filter.created_at_lt_millis {
body["created_at_lt_millis"] = end.into();
}
if let Some(token) = options.page_token {
body["page_token"] = token.into();
}
@@ -1999,7 +2047,7 @@ mod tests {
use rstest::rstest;
use crate::connection::ConnectBuilder;
use crate::database::Database;
use crate::database::{Database, JobListFilter, JobListState};
use crate::error::{JobFailure, RemoteErrorMetadata, RemoteResourceIdentity};
use crate::graph::{EdgeTable, Endpoint, NodeTable, PropertyGraphDefinition};
use crate::materialized_view::CreateMaterializedViewRequest;
@@ -4133,7 +4181,10 @@ mod tests {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
.unwrap();
assert_eq!(body, serde_json::json!({"page_token": token, "limit": 2}));
assert_eq!(
body,
serde_json::json!({"database": "default", "page_token": token, "limit": 2})
);
} else {
let ns = "team";
assert_eq!(
@@ -4336,6 +4387,84 @@ mod tests {
));
}
#[tokio::test]
async fn test_list_jobs_filters_every_page_without_eager_fetching() {
let count = Arc::new(AtomicUsize::new(0));
let seen = count.clone();
let conn = Connection::new_with_handler(move |request| {
let page = seen.fetch_add(1, Ordering::SeqCst);
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
let mut expected = serde_json::json!({
"database": "default",
"table_ref": {
"db_name": "default",
"table_id": {"namespace_path": ["team", "project"], "table_name": "events"},
},
"state": "in_progress",
"job_types": ["column_refresh", "materialized_view_refresh"],
"created_at_gte_millis": 100,
"created_at_lt_millis": 200,
"limit": 1,
});
if page > 0 {
expected["page_token"] = "next".into();
}
assert_eq!(body, expected);
http::Response::builder().status(200).body(serde_json::json!({
"jobs": [{"job_id": format!("job-{page}"), "table": "events", "job_type": "column_refresh", "state": "in_progress", "created_at_millis": 100}],
"page_token": (page == 0).then_some("next"),
}).to_string()).unwrap()
});
let filter = JobListFilter {
table: Some("events".into()),
namespace_path: vec!["team".into(), "project".into()],
state: Some(JobListState::InProgress),
job_types: vec!["column_refresh".into(), "materialized_view_refresh".into()],
created_at_gte_millis: Some(100),
created_at_lt_millis: Some(200),
};
let mut jobs =
conn.list_jobs_filtered(filter.clone(), ListingOptions::default().page_limit(1));
assert_eq!(count.load(Ordering::SeqCst), 0);
assert_eq!(jobs.try_next().await.unwrap().unwrap().job_id, "job-0");
assert_eq!(count.load(Ordering::SeqCst), 1);
let resumed = conn.list_jobs_filtered(
filter,
ListingOptions::default()
.page_limit(1)
.page_token(jobs.page_token().unwrap()),
);
let remaining = resumed.try_collect::<Vec<_>>().await.unwrap();
assert_eq!(remaining.len(), 1);
assert_eq!(remaining[0].job_id, "job-1");
assert_eq!(count.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn test_list_jobs_invalid_filters_never_reach_server() {
let conn = Connection::new_with_handler(|_| -> http::Response<String> {
panic!("invalid filters must not reach the server")
});
let namespace_only = JobListFilter {
namespace_path: vec!["team".into()],
..Default::default()
};
let invalid_time = JobListFilter {
created_at_gte_millis: Some(200),
created_at_lt_millis: Some(100),
..Default::default()
};
for filter in [namespace_only, invalid_time] {
let mut jobs = conn.list_jobs_filtered(filter, ListingOptions::default());
assert!(matches!(
jobs.try_next().await,
Err(Error::InvalidInput { .. })
));
assert!(jobs.try_next().await.unwrap().is_none());
}
}
#[tokio::test]
async fn test_list_jobs_does_not_truncate_after_100_pages() {
let conn = Connection::new_with_handler(|request| {