mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat: expose server-side job listing filters (#5041)
Expose server-side job filters and lazy Listing indexing, slicing, and length helpers. Document cache, cursor, and SDK/server compatibility behavior.
This commit is contained in:
@@ -177,14 +177,51 @@ for info in db.list_jobs(page_limit=50):
|
||||
live `Job` handles. Open an ID to inspect its full record. `page_limit` bounds
|
||||
each server request, not the total yielded items, and `page_token` resumes a
|
||||
listing. It is a lazy iterator: iteration performs requests and can raise
|
||||
request errors. To materialize it, use `list(db.list_jobs(...))`; it does not
|
||||
support list indexing, `len()`, or a `.to_list()` method.
|
||||
request errors. Use `list(db.list_jobs(...))` to collect all remaining results;
|
||||
there is no `.to_list()` method.
|
||||
|
||||
`Listing.num_page_results()` reports cached items remaining in the current
|
||||
page and `Listing.page_token()` reports the next request's token. Drain the
|
||||
cached items before saving that token, or resuming can skip them. A `None`
|
||||
token after the final page can coexist with unread cached items. A resumed
|
||||
listing is not a transactionally frozen view of jobs submitted in the meantime.
|
||||
<Note>
|
||||
Job filters and synchronous `Listing` indexing, slicing, and `len()` require
|
||||
a Python SDK release that includes these APIs. Older SDKs expose only
|
||||
`page_token` and `page_limit`; collect results with `list(...)` before indexing
|
||||
or counting them. Filtering also requires a matching Enterprise server that
|
||||
supports the requested filters.
|
||||
</Note>
|
||||
|
||||
Filter jobs on the server before pagination with `table`, `state`, `job_type`
|
||||
or `job_types`, and creation-time bounds. `table` names a table in
|
||||
`namespace_path` (the root by default) and includes all its branches. Requests
|
||||
stay scoped to the connected database. Creation times are Unix milliseconds:
|
||||
`created_at_gte_millis` is inclusive and `created_at_lt_millis` is exclusive.
|
||||
Choose either `job_type` or `job_types`.
|
||||
|
||||
The `state` filter uses registry states: `in_progress`, `succeeded`, `failed`,
|
||||
`canceled`, and `timed_out`. These differ from the SDK lifecycle labels such
|
||||
as `running` and `finished` returned in `JobInfo.state`.
|
||||
|
||||
```python
|
||||
jobs = db.list_jobs(table="events", state="in_progress", page_limit=5)
|
||||
first_five = jobs[:5]
|
||||
for info in first_five:
|
||||
print(info.job_id, info.state)
|
||||
```
|
||||
|
||||
Synchronous indexing and slicing refer to the remaining results without
|
||||
consuming them. A bounded forward slice such as `jobs[:5]` fetches only enough
|
||||
pages to reach its stop. `len(jobs)` returns the remaining count by fetching
|
||||
and caching every remaining result. Negative indexes and open-ended or reverse
|
||||
slices also fetch and cache all remaining results. These operations can raise
|
||||
request errors; later iteration drains their cache. `AsyncListing` remains an
|
||||
async iterator: use `async for` to read it.
|
||||
|
||||
`Listing.num_page_results()` reports cached items available without another
|
||||
request, including items retained by indexing, slicing, or counting.
|
||||
`Listing.page_token()` reports the next request's token. Drain all cached
|
||||
items before saving that token, or resuming can skip them. Resume with the same
|
||||
filters, for example `db.list_jobs(table="events", state="in_progress",
|
||||
page_token=saved_token, page_limit=5)`. A `None` token after the final page can
|
||||
coexist with unread cached items. A resumed listing is not a transactionally
|
||||
frozen view of jobs submitted in the meantime.
|
||||
|
||||
`events.schema` describes the Arrow response. Use `events.to_pylist()` for
|
||||
Python row dictionaries. Common event columns are `event_seq` (int64), `state`
|
||||
|
||||
@@ -2318,10 +2318,30 @@ describe("remote connection jobs surface", () => {
|
||||
queue_reason: "Waiting for Function worker capacity",
|
||||
});
|
||||
const descriptions = [
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 3000, unknownFleet)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(20, 80, 2000, knownFleet)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(40, 60, 1000, unknownFleet)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(60, 0, 0, unknownFleet)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
|
||||
20,
|
||||
80,
|
||||
3000,
|
||||
unknownFleet,
|
||||
)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
|
||||
20,
|
||||
80,
|
||||
2000,
|
||||
knownFleet,
|
||||
)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"queue_reason":"Waiting for Function worker capacity","progress":${progress(
|
||||
40,
|
||||
60,
|
||||
1000,
|
||||
unknownFleet,
|
||||
)}}`,
|
||||
`{"job_id":"job-wait","job_type":"refresh_column","job_state":"DONE","creation_ms":1000,"result":{},"progress":${progress(
|
||||
60,
|
||||
0,
|
||||
0,
|
||||
unknownFleet,
|
||||
)}}`,
|
||||
];
|
||||
let describeCount = 0;
|
||||
|
||||
@@ -2393,12 +2413,24 @@ describe("remote connection jobs surface", () => {
|
||||
it("does not derive ETA across a zero-rate, epoch, or pause boundary", async () => {
|
||||
const now = Date.now();
|
||||
const descriptions = [
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 5000}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${now - 4000}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 3000}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 2000}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1500}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${now - 1000}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${
|
||||
now - 5000
|
||||
}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":20,"rows_skipped":0,"rows_remaining":80,"fragments_planned":4,"execution_epoch":1,"sample_at_millis":${
|
||||
now - 4000
|
||||
}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":40,"rows_skipped":0,"rows_remaining":60,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
|
||||
now - 3000
|
||||
}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"phase":"initializing","progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
|
||||
now - 2000
|
||||
}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"paused_at_millis":5000,"progress":{"rows_planned":100,"rows_computed":60,"rows_skipped":0,"rows_remaining":40,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
|
||||
now - 1500
|
||||
}}}`,
|
||||
`{"job_id":"job-eta","job_type":"refresh_column","job_state":"IN_PROGRESS","creation_ms":1000,"progress":{"rows_planned":100,"rows_computed":80,"rows_skipped":0,"rows_remaining":20,"fragments_planned":4,"execution_epoch":2,"sample_at_millis":${
|
||||
now - 1000
|
||||
}}}`,
|
||||
];
|
||||
|
||||
await withMockDatabase(
|
||||
@@ -2574,6 +2606,7 @@ describe("lazy resource listings", () => {
|
||||
if (kind === "jobs") {
|
||||
expect(url.pathname).toBe("/v1/jobs/list");
|
||||
expect(request.body).toEqual({
|
||||
database: "dev",
|
||||
// biome-ignore lint/style/useNamingConvention: server wire format
|
||||
page_token: token,
|
||||
limit: 2,
|
||||
|
||||
@@ -300,7 +300,19 @@ class Connection(object):
|
||||
page_limit: Optional[int] = None,
|
||||
) -> NameListing: ...
|
||||
def list_jobs(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
self,
|
||||
*,
|
||||
page_token: Optional[str] = None,
|
||||
page_limit: Optional[int] = None,
|
||||
table: Optional[str] = None,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
state: Optional[
|
||||
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
|
||||
] = None,
|
||||
job_type: Optional[str] = None,
|
||||
job_types: Optional[List[str]] = None,
|
||||
created_at_gte_millis: Optional[int] = None,
|
||||
created_at_lt_millis: Optional[int] = None,
|
||||
) -> JobListing: ...
|
||||
async def cancel_job(self, job_id: str) -> bool: ...
|
||||
async def pause_job(self, job_id: str) -> str: ...
|
||||
|
||||
+119
-17
@@ -1602,13 +1602,39 @@ class DBConnection(EnforceOverrides):
|
||||
)
|
||||
|
||||
def list_jobs(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
self,
|
||||
*,
|
||||
page_token: Optional[str] = None,
|
||||
page_limit: Optional[int] = None,
|
||||
table: Optional[str] = None,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
state: Optional[
|
||||
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
|
||||
] = None,
|
||||
job_type: Optional[str] = None,
|
||||
job_types: Optional[List[str]] = None,
|
||||
created_at_gte_millis: Optional[int] = None,
|
||||
created_at_lt_millis: Optional[int] = None,
|
||||
) -> Listing[JobInfo]:
|
||||
"""List server-side jobs across the database's tables.
|
||||
"""List server-side jobs, optionally filtered before pagination.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
|
||||
``page_token`` resumes from a saved token. Requests and errors occur during
|
||||
iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
``table`` names a table in ``namespace_path`` (the root by default).
|
||||
Listings include all branches of the table. A namespace filter requires
|
||||
``table``.
|
||||
``state`` uses registry states such as ``in_progress``, ``succeeded``,
|
||||
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
|
||||
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
|
||||
Creation-time bounds are Unix milliseconds, inclusive at
|
||||
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
|
||||
Resume a ``page_token`` with the same filters. Requests and errors occur
|
||||
during iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
|
||||
For example, to inspect up to five jobs without fetching all history::
|
||||
|
||||
jobs = db.list_jobs(table="events", page_limit=5)
|
||||
first_five = jobs[:5]
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
_append_doc_url(
|
||||
@@ -2799,16 +2825,52 @@ class LanceDBConnection(DBConnection):
|
||||
|
||||
@override
|
||||
def list_jobs(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
self,
|
||||
*,
|
||||
page_token: Optional[str] = None,
|
||||
page_limit: Optional[int] = None,
|
||||
table: Optional[str] = None,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
state: Optional[
|
||||
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
|
||||
] = None,
|
||||
job_type: Optional[str] = None,
|
||||
job_types: Optional[List[str]] = None,
|
||||
created_at_gte_millis: Optional[int] = None,
|
||||
created_at_lt_millis: Optional[int] = None,
|
||||
) -> Listing[JobInfo]:
|
||||
"""List server-side jobs across the database's tables.
|
||||
"""List server-side jobs, optionally filtered before pagination.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
|
||||
``page_token`` resumes from a saved token. Requests and errors occur during
|
||||
iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
``table`` names a table in ``namespace_path`` (the root by default).
|
||||
Listings include all branches of the table. A namespace filter requires
|
||||
``table``.
|
||||
``state`` uses registry states such as ``in_progress``, ``succeeded``,
|
||||
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
|
||||
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
|
||||
Creation-time bounds are Unix milliseconds, inclusive at
|
||||
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
|
||||
Resume a ``page_token`` with the same filters. Requests and errors occur
|
||||
during iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
|
||||
For example, to inspect up to five jobs without fetching all history::
|
||||
|
||||
jobs = db.list_jobs(table="events", page_limit=5)
|
||||
first_five = jobs[:5]
|
||||
"""
|
||||
return Listing(
|
||||
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
|
||||
self._conn.list_jobs(
|
||||
page_token=page_token,
|
||||
page_limit=page_limit,
|
||||
table=table,
|
||||
namespace_path=namespace_path,
|
||||
state=state,
|
||||
job_type=job_type,
|
||||
job_types=job_types,
|
||||
created_at_gte_millis=created_at_gte_millis,
|
||||
created_at_lt_millis=created_at_lt_millis,
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
@@ -4375,16 +4437,56 @@ class AsyncConnection(object):
|
||||
)
|
||||
|
||||
def list_jobs(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
self,
|
||||
*,
|
||||
page_token: Optional[str] = None,
|
||||
page_limit: Optional[int] = None,
|
||||
table: Optional[str] = None,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
state: Optional[
|
||||
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
|
||||
] = None,
|
||||
job_type: Optional[str] = None,
|
||||
job_types: Optional[List[str]] = None,
|
||||
created_at_gte_millis: Optional[int] = None,
|
||||
created_at_lt_millis: Optional[int] = None,
|
||||
) -> AsyncListing[JobInfo]:
|
||||
"""List server-side jobs across the database's tables.
|
||||
"""List server-side jobs, optionally filtered before pagination.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
|
||||
``page_token`` resumes from a saved token. Requests and errors occur during
|
||||
iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
``table`` names a table in ``namespace_path`` (the root by default).
|
||||
Listings include all branches of the table. A namespace filter requires
|
||||
``table``.
|
||||
``state`` uses registry states such as ``in_progress``, ``succeeded``,
|
||||
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
|
||||
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
|
||||
Creation-time bounds are Unix milliseconds, inclusive at
|
||||
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
|
||||
Resume a ``page_token`` with the same filters. Requests and errors occur
|
||||
during iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
|
||||
For example, to inspect up to five jobs without fetching all history::
|
||||
|
||||
jobs = db.list_jobs(table="events", page_limit=5)
|
||||
first_five = []
|
||||
async for job in jobs:
|
||||
first_five.append(job)
|
||||
if len(first_five) == 5:
|
||||
break
|
||||
"""
|
||||
return AsyncListing(
|
||||
self._inner.list_jobs(page_token=page_token, page_limit=page_limit)
|
||||
self._inner.list_jobs(
|
||||
page_token=page_token,
|
||||
page_limit=page_limit,
|
||||
table=table,
|
||||
namespace_path=namespace_path,
|
||||
state=state,
|
||||
job_type=job_type,
|
||||
job_types=job_types,
|
||||
created_at_gte_millis=created_at_gte_millis,
|
||||
created_at_lt_millis=created_at_lt_millis,
|
||||
)
|
||||
)
|
||||
|
||||
async def cancel_job(self, job_id: str) -> bool:
|
||||
|
||||
@@ -6,7 +6,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
|
||||
from collections import deque
|
||||
from itertools import islice
|
||||
from operator import index
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncIterator,
|
||||
Callable,
|
||||
Generic,
|
||||
Iterator,
|
||||
Optional,
|
||||
TypeVar,
|
||||
overload,
|
||||
)
|
||||
|
||||
from .background_loop import LOOP
|
||||
|
||||
@@ -59,29 +71,85 @@ class Listing(Iterator[T], Generic[T]):
|
||||
print(name)
|
||||
|
||||
Use ``list(connection.list_views())`` to collect all results.
|
||||
|
||||
Indexes and slices refer to the remaining items and do not consume them.
|
||||
A bounded forward slice, such as ``listing[:5]``, fetches only enough pages
|
||||
to reach its stop. ``len(listing)``, negative indexes, and open-ended or
|
||||
reverse slices fetch and cache all remaining results. Iteration then drains
|
||||
that cache. These operations may perform requests and raise request errors.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: AsyncListing[T]):
|
||||
self._inner = inner
|
||||
self._buffer: deque[T] = deque()
|
||||
|
||||
def __iter__(self) -> Listing[T]:
|
||||
return self
|
||||
|
||||
def __next__(self) -> T:
|
||||
if self._buffer:
|
||||
return self._buffer.popleft()
|
||||
try:
|
||||
return LOOP.run(self._inner.__anext__())
|
||||
except StopAsyncIteration:
|
||||
raise StopIteration from None
|
||||
|
||||
def _fill(self, count: Optional[int]) -> None:
|
||||
while count is None or len(self._buffer) < count:
|
||||
try:
|
||||
item = LOOP.run(self._inner.__anext__())
|
||||
except StopAsyncIteration:
|
||||
return
|
||||
self._buffer.append(item)
|
||||
|
||||
def __len__(self) -> int:
|
||||
"""Fetch and cache every remaining item, returning the remaining count."""
|
||||
self._fill(None)
|
||||
return len(self._buffer)
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
self._fill(1)
|
||||
return bool(self._buffer)
|
||||
|
||||
@overload
|
||||
def __getitem__(self, key: int) -> T: ...
|
||||
|
||||
@overload
|
||||
def __getitem__(self, key: slice) -> list[T]: ...
|
||||
|
||||
def __getitem__(self, key: int | slice) -> T | list[T]:
|
||||
"""Read remaining items without consuming them; bounded slices stay lazy."""
|
||||
if isinstance(key, slice):
|
||||
start = 0 if key.start is None else index(key.start)
|
||||
stop = None if key.stop is None else index(key.stop)
|
||||
step = 1 if key.step is None else index(key.step)
|
||||
if step == 0:
|
||||
raise ValueError("slice step cannot be zero")
|
||||
if start >= 0 and stop is not None and stop >= 0 and step > 0:
|
||||
if start >= stop:
|
||||
return []
|
||||
self._fill(stop)
|
||||
start, stop, step = key.indices(len(self._buffer))
|
||||
return list(
|
||||
islice(self._buffer, start, stop, min(step, len(self._buffer) + 1))
|
||||
)
|
||||
self._fill(None)
|
||||
return list(self._buffer)[key]
|
||||
position = index(key)
|
||||
self._fill(None if position < 0 else position + 1)
|
||||
return self._buffer[position]
|
||||
|
||||
def num_page_results(self) -> int:
|
||||
"""Return the number of cached items available without another request."""
|
||||
return self._inner.num_page_results()
|
||||
return len(self._buffer) + self._inner.num_page_results()
|
||||
|
||||
def page_token(self) -> Optional[str]:
|
||||
"""Return the next request's token.
|
||||
|
||||
See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token]
|
||||
for initial, cached, final-page, and error behavior.
|
||||
Indexing, slicing, and counting can also advance this token; drain all
|
||||
``num_page_results()`` cached items before resuming from it.
|
||||
"""
|
||||
return self._inner.page_token()
|
||||
|
||||
|
||||
@@ -1324,16 +1324,52 @@ class RemoteDBConnection(DBConnection):
|
||||
|
||||
@override
|
||||
def list_jobs(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
self,
|
||||
*,
|
||||
page_token: Optional[str] = None,
|
||||
page_limit: Optional[int] = None,
|
||||
table: Optional[str] = None,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
state: Optional[
|
||||
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
|
||||
] = None,
|
||||
job_type: Optional[str] = None,
|
||||
job_types: Optional[List[str]] = None,
|
||||
created_at_gte_millis: Optional[int] = None,
|
||||
created_at_lt_millis: Optional[int] = None,
|
||||
) -> Listing[JobInfo]:
|
||||
"""List server-side jobs across the database's tables.
|
||||
"""List server-side jobs, optionally filtered before pagination.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
|
||||
``page_token`` resumes from a saved token. Requests and errors occur during
|
||||
iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
``table`` names a table in ``namespace_path`` (the root by default).
|
||||
Listings include all branches of the table. A namespace filter requires
|
||||
``table``.
|
||||
``state`` uses registry states such as ``in_progress``, ``succeeded``,
|
||||
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
|
||||
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
|
||||
Creation-time bounds are Unix milliseconds, inclusive at
|
||||
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
|
||||
|
||||
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
|
||||
Resume a ``page_token`` with the same filters. Requests and errors occur
|
||||
during iteration. See [pagination state][lancedb.listing.AsyncListing].
|
||||
|
||||
For example, to inspect up to five jobs without fetching all history::
|
||||
|
||||
jobs = db.list_jobs(table="events", page_limit=5)
|
||||
first_five = jobs[:5]
|
||||
"""
|
||||
return Listing(
|
||||
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
|
||||
self._conn.list_jobs(
|
||||
page_token=page_token,
|
||||
page_limit=page_limit,
|
||||
table=table,
|
||||
namespace_path=namespace_path,
|
||||
state=state,
|
||||
job_type=job_type,
|
||||
job_types=job_types,
|
||||
created_at_gte_millis=created_at_gte_millis,
|
||||
created_at_lt_millis=created_at_lt_millis,
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
|
||||
@@ -268,7 +268,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous)
|
||||
if kind == "jobs":
|
||||
assert verb == "POST"
|
||||
assert path.path == "/v1/jobs/list"
|
||||
assert body == {"page_token": token, "limit": 2}
|
||||
assert body == {"database": "dev", "page_token": token, "limit": 2}
|
||||
else:
|
||||
namespace = "team"
|
||||
resource = kind.removesuffix("s")
|
||||
@@ -326,3 +326,179 @@ async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_l
|
||||
with pytest.raises(ValueError, match="limit"):
|
||||
await items.__anext__()
|
||||
assert requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("namespace", [[], ["team", "project"]])
|
||||
@pytest.mark.parametrize("multiple_types", [False, True])
|
||||
async def test_job_filters_persist_across_pages_and_resume(
|
||||
listing_server, asynchronous, namespace, multiple_types
|
||||
):
|
||||
endpoint, requests, responses = listing_server
|
||||
responses.extend(
|
||||
(200, response_page("jobs", names, token))
|
||||
for names, token in [(["a"], "empty"), ([], "last"), (["b"], None)]
|
||||
)
|
||||
db = await connection(endpoint, asynchronous)
|
||||
types = ["column_refresh", "materialized_view_refresh"]
|
||||
filters = dict(
|
||||
table="events",
|
||||
namespace_path=namespace,
|
||||
state="in_progress",
|
||||
created_at_gte_millis=100,
|
||||
created_at_lt_millis=200,
|
||||
**({"job_types": types} if multiple_types else {"job_type": types[0]}),
|
||||
)
|
||||
items = db.list_jobs(page_limit=1, **filters)
|
||||
assert requests == []
|
||||
assert (await advance(items, asynchronous)).job_id == "a"
|
||||
assert len(requests) == 1
|
||||
resumed = db.list_jobs(page_token=items.page_token(), page_limit=1, **filters)
|
||||
assert (await advance(resumed, asynchronous)).job_id == "b"
|
||||
assert resumed.page_token() is None
|
||||
assert len(requests) == 3
|
||||
expected = dict(
|
||||
database="dev",
|
||||
limit=1,
|
||||
table_ref={
|
||||
"db_name": "dev",
|
||||
"table_id": {"namespace_path": namespace, "table_name": "events"},
|
||||
},
|
||||
state="in_progress",
|
||||
job_types=types if multiple_types else types[:1],
|
||||
created_at_gte_millis=100,
|
||||
created_at_lt_millis=200,
|
||||
)
|
||||
for (_, path, body), token in zip(requests, [None, "empty", "last"]):
|
||||
assert path == "/v1/jobs/list"
|
||||
assert body == {**expected, **({"page_token": token} if token else {})}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"filters",
|
||||
[
|
||||
{},
|
||||
{"state": "failed"},
|
||||
{"job_types": ["column_refresh"]},
|
||||
{"created_at_lt_millis": 200},
|
||||
],
|
||||
)
|
||||
async def test_job_listing_without_table_stays_in_connected_database(
|
||||
listing_server, asynchronous, filters
|
||||
):
|
||||
endpoint, requests, responses = listing_server
|
||||
responses.append((200, response_page("jobs", ["a"], None)))
|
||||
db = await connection(endpoint, asynchronous)
|
||||
items = db.list_jobs(**filters)
|
||||
assert (await advance(items, asynchronous)).job_id == "a"
|
||||
assert requests[0][2] == {"database": "dev", **filters}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"filters, message",
|
||||
[
|
||||
({"namespace_path": ["team"]}, "requires a table"),
|
||||
({"created_at_gte_millis": 200, "created_at_lt_millis": 200}, "less than"),
|
||||
({"created_at_gte_millis": 201, "created_at_lt_millis": 200}, "less than"),
|
||||
({"job_type": "column_refresh", "job_types": []}, "only one"),
|
||||
({"state": "unknown"}, "state must be"),
|
||||
],
|
||||
)
|
||||
async def test_invalid_job_filters_do_not_fetch(
|
||||
listing_server, asynchronous, filters, message
|
||||
):
|
||||
endpoint, requests, _ = listing_server
|
||||
db = await connection(endpoint, asynchronous)
|
||||
with pytest.raises(ValueError, match=message):
|
||||
items = db.list_jobs(**filters)
|
||||
await advance(items, asynchronous)
|
||||
assert requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_listing_slice_and_index_fetch_only_needed_pages(listing_server):
|
||||
endpoint, requests, responses = listing_server
|
||||
responses.extend(
|
||||
(200, response_page("jobs", names, token))
|
||||
for names, token in [(["a", "b"], "second"), (["c", "d"], "last")]
|
||||
)
|
||||
db = await connection(endpoint, False)
|
||||
items = db.list_jobs(table="events", page_limit=2)
|
||||
assert items[:0] == []
|
||||
assert items[5:2] == []
|
||||
assert requests == []
|
||||
assert [job.job_id for job in items[:2]] == ["a", "b"]
|
||||
assert len(requests) == 1
|
||||
assert items.num_page_results() == 2
|
||||
assert items.page_token() == "second"
|
||||
assert items[1].job_id == "b"
|
||||
assert next(items).job_id == "a"
|
||||
assert [job.job_id for job in items[:2]] == ["b", "c"]
|
||||
assert len(requests) == 2
|
||||
assert items.num_page_results() == 3
|
||||
assert items.page_token() == "last"
|
||||
assert [job.job_id for job in items[0:3:2][:1]] == ["b"]
|
||||
assert len(requests) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_listing_length_and_slices_preserve_remaining_items(listing_server):
|
||||
endpoint, requests, responses = listing_server
|
||||
responses.extend(
|
||||
(200, response_page("jobs", names, token))
|
||||
for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], None)]
|
||||
)
|
||||
db = await connection(endpoint, False)
|
||||
items = db.list_jobs(page_limit=2)
|
||||
assert bool(items)
|
||||
assert len(requests) == 1
|
||||
assert next(items).job_id == "a"
|
||||
assert len(items) == 3
|
||||
assert len(requests) == 3
|
||||
assert items.num_page_results() == 3
|
||||
assert items.page_token() is None
|
||||
assert items[-1].job_id == "d"
|
||||
assert [job.job_id for job in items[::-1]] == ["d", "c", "b"]
|
||||
assert [job.job_id for job in items[1:]] == ["c", "d"]
|
||||
assert [job.job_id for job in items[:100:2]] == ["b", "d"]
|
||||
assert [job.job_id for job in items[: 2**100 : 2**100]] == ["b"]
|
||||
with pytest.raises(IndexError):
|
||||
_ = items[3]
|
||||
assert [job.job_id for job in items] == ["b", "c", "d"]
|
||||
assert len(items) == 0
|
||||
assert not items
|
||||
assert len(requests) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_listing_invalid_slice_does_not_fetch(listing_server):
|
||||
endpoint, requests, _ = listing_server
|
||||
db = await connection(endpoint, False)
|
||||
items = db.list_jobs()
|
||||
with pytest.raises(ValueError, match="step"):
|
||||
_ = items[::0]
|
||||
with pytest.raises(TypeError):
|
||||
_ = items["invalid"]
|
||||
with pytest.raises(TypeError):
|
||||
_ = items[:1.5]
|
||||
assert requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_listing_slice_failure_preserves_cache_and_retry_token(listing_server):
|
||||
endpoint, requests, responses = listing_server
|
||||
responses.extend([(200, response_page("jobs", ["a", "b"], "retry")), (401, {})])
|
||||
db = await connection(endpoint, False)
|
||||
items = db.list_jobs(page_limit=2)
|
||||
with pytest.raises(HttpError):
|
||||
_ = items[:3]
|
||||
assert items.page_token() == "retry"
|
||||
assert items.num_page_results() == 2
|
||||
assert [job.job_id for job in items[:2]] == ["a", "b"]
|
||||
assert [job.job_id for job in items] == ["a", "b"]
|
||||
assert len(requests) == 2
|
||||
|
||||
@@ -23,7 +23,7 @@ use lancedb::{
|
||||
connection::Connection as LanceConnection,
|
||||
connection::NamespaceClientPushdownOperation,
|
||||
database::namespace::LanceNamespaceDatabase,
|
||||
database::{CreateTableMode, Database, ReadConsistency},
|
||||
database::{CreateTableMode, Database, JobListFilter, JobListState, ReadConsistency},
|
||||
function::{GpuClass as LanceGpuClass, RefreshColumnResult},
|
||||
graph::PropertyGraphDefinition,
|
||||
materialized_view::CreateMaterializedViewBuilder,
|
||||
@@ -1319,15 +1319,47 @@ impl Connection {
|
||||
))
|
||||
}
|
||||
|
||||
#[pyo3(signature = (*, page_token=None, page_limit=None))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (*, page_token=None, page_limit=None, table=None, namespace_path=None, state=None, job_type=None, job_types=None, created_at_gte_millis=None, created_at_lt_millis=None))]
|
||||
pub fn list_jobs(
|
||||
&self,
|
||||
page_token: Option<String>,
|
||||
page_limit: Option<u32>,
|
||||
table: Option<String>,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
state: Option<String>,
|
||||
job_type: Option<String>,
|
||||
job_types: Option<Vec<String>>,
|
||||
created_at_gte_millis: Option<i64>,
|
||||
created_at_lt_millis: Option<i64>,
|
||||
) -> PyResult<crate::listing::JobListing> {
|
||||
if job_type.is_some() && job_types.is_some() {
|
||||
return Err(PyValueError::new_err(
|
||||
"set only one of job_type and job_types",
|
||||
));
|
||||
}
|
||||
let mut filter = JobListFilter::default();
|
||||
filter.table = table;
|
||||
filter.namespace_path = namespace_path.unwrap_or_default();
|
||||
filter.state = state
|
||||
.as_deref()
|
||||
.map(|state| match state {
|
||||
"in_progress" => Ok(JobListState::InProgress),
|
||||
"succeeded" => Ok(JobListState::Succeeded),
|
||||
"failed" => Ok(JobListState::Failed),
|
||||
"canceled" => Ok(JobListState::Canceled),
|
||||
"timed_out" => Ok(JobListState::TimedOut),
|
||||
_ => Err(PyValueError::new_err(
|
||||
"state must be one of: in_progress, succeeded, failed, canceled, timed_out",
|
||||
)),
|
||||
})
|
||||
.transpose()?;
|
||||
filter.job_types = job_types.unwrap_or_else(|| job_type.into_iter().collect());
|
||||
filter.created_at_gte_millis = created_at_gte_millis;
|
||||
filter.created_at_lt_millis = created_at_lt_millis;
|
||||
Ok(crate::listing::JobListing::new(
|
||||
self.get_inner()?
|
||||
.list_jobs(crate::listing::options(page_token, page_limit)),
|
||||
.list_jobs_filtered(filter, crate::listing::options(page_token, page_limit)),
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -23,8 +23,8 @@ use crate::connection::create_table::CreateTableBuilder;
|
||||
use crate::data::scannable::Scannable;
|
||||
use crate::database::listing::ListingDatabase;
|
||||
use crate::database::{
|
||||
CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, OpenTableRequest,
|
||||
PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
|
||||
CloneTableRequest, Database, DatabaseOptions, ExecuteQueryRequest, JobInfo, JobListFilter,
|
||||
OpenTableRequest, PauseJobStatus, ReadConsistency, ResumeJobStatus, TableNamesRequest,
|
||||
};
|
||||
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
|
||||
use crate::error::{Error, Result};
|
||||
@@ -1366,6 +1366,37 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
/// List jobs using server-side filters, without scanning all history in the client.
|
||||
///
|
||||
/// Filters are preserved on every page, including when resuming with a saved
|
||||
/// token. Requests and errors occur during iteration, as with [`Self::list_jobs`].
|
||||
///
|
||||
/// ```
|
||||
/// # async fn example(connection: &lancedb::Connection) -> lancedb::Result<()> {
|
||||
/// use futures::TryStreamExt;
|
||||
/// use lancedb::{database::JobListFilter, listing::ListingOptions};
|
||||
/// let mut filter = JobListFilter::default();
|
||||
/// filter.table = Some("events".into());
|
||||
/// let mut jobs = connection.list_jobs_filtered(filter, ListingOptions::default().page_limit(5));
|
||||
/// if let Some(job) = jobs.try_next().await? {
|
||||
/// println!("{}", job.job_id);
|
||||
/// }
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn list_jobs_filtered(
|
||||
&self,
|
||||
filter: JobListFilter,
|
||||
options: ListingOptions,
|
||||
) -> Listing<JobInfo> {
|
||||
let database = self.database().clone();
|
||||
Listing::new(options, move |options| {
|
||||
let database = database.clone();
|
||||
let filter = filter.clone();
|
||||
async move { database.list_jobs_filtered(filter, options).await }
|
||||
})
|
||||
}
|
||||
|
||||
/// Request cancellation of a server-side job by id. Returns true if the
|
||||
/// server accepted the cancellation, false if no such job exists.
|
||||
pub async fn cancel_job(&self, job_id: impl AsRef<str>) -> Result<bool> {
|
||||
|
||||
@@ -238,6 +238,45 @@ pub enum ReadConsistency {
|
||||
Strong,
|
||||
}
|
||||
|
||||
/// Registry lifecycle states accepted when filtering jobs.
|
||||
#[derive(Clone, Copy, Debug, serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum JobListState {
|
||||
/// A job that has not reached a terminal state.
|
||||
InProgress,
|
||||
/// A job that completed successfully.
|
||||
Succeeded,
|
||||
/// A job that failed.
|
||||
Failed,
|
||||
/// A job that was canceled.
|
||||
Canceled,
|
||||
/// A job that exceeded its timeout.
|
||||
TimedOut,
|
||||
}
|
||||
|
||||
/// Server-side filters applied before job pagination.
|
||||
///
|
||||
/// Continuation tokens must be resumed with the same filters. Creation times
|
||||
/// are Unix timestamps in milliseconds; the lower bound is inclusive and the
|
||||
/// upper bound is exclusive.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
#[non_exhaustive]
|
||||
pub struct JobListFilter {
|
||||
/// Table name within `namespace_path`, including all its branches; absent means all tables.
|
||||
pub table: Option<String>,
|
||||
/// Namespace of `table`; requires a table name.
|
||||
pub namespace_path: Vec<String>,
|
||||
/// Registry state, such as `in_progress`, `succeeded`, or `failed`.
|
||||
/// These are server states, rather than the labels in [`JobInfo::state`].
|
||||
pub state: Option<JobListState>,
|
||||
/// Include jobs of any of these types; empty means all types.
|
||||
pub job_types: Vec<String>,
|
||||
/// Inclusive creation-time lower bound.
|
||||
pub created_at_gte_millis: Option<i64>,
|
||||
/// Exclusive creation-time upper bound.
|
||||
pub created_at_lt_millis: Option<i64>,
|
||||
}
|
||||
|
||||
/// A row from [`Database::list_jobs`]: one server-side job (index build,
|
||||
/// compaction, column refresh, ...).
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -781,6 +820,14 @@ pub trait Database:
|
||||
async fn list_jobs(&self, _options: ListingOptions) -> Result<ListingPage<JobInfo>> {
|
||||
job_op_not_supported("list_jobs")
|
||||
}
|
||||
/// Fetch one page of jobs matching server-side filters.
|
||||
async fn list_jobs_filtered(
|
||||
&self,
|
||||
_filter: JobListFilter,
|
||||
_options: ListingOptions,
|
||||
) -> Result<ListingPage<JobInfo>> {
|
||||
job_op_not_supported("list_jobs_filtered")
|
||||
}
|
||||
/// Request cancellation of a job by id. Returns true if the server
|
||||
/// accepted the cancellation, false if no such job exists. Cancelling an
|
||||
/// already-terminal job is a no-op success.
|
||||
|
||||
@@ -22,7 +22,8 @@ use lance_namespace::models::{
|
||||
use crate::Error;
|
||||
use crate::database::{
|
||||
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions,
|
||||
ExecuteQueryRequest, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
|
||||
ExecuteQueryRequest, JobInfo, JobListFilter, OpenTableRequest, ReadConsistency,
|
||||
TableNamesRequest,
|
||||
};
|
||||
use crate::error::Result;
|
||||
use crate::function::{FunctionRegistrationRequest, FunctionVersion, GpuClass};
|
||||
@@ -1290,7 +1291,54 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
}
|
||||
|
||||
async fn list_jobs(&self, options: ListingOptions) -> Result<ListingPage<JobInfo>> {
|
||||
let mut body = serde_json::json!({});
|
||||
self.list_jobs_filtered(JobListFilter::default(), options)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_jobs_filtered(
|
||||
&self,
|
||||
filter: JobListFilter,
|
||||
options: ListingOptions,
|
||||
) -> Result<ListingPage<JobInfo>> {
|
||||
if self.db_name.is_empty() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "list_jobs requires a connected database".into(),
|
||||
});
|
||||
}
|
||||
if filter.table.is_none() && !filter.namespace_path.is_empty() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "namespace_path requires a table filter".into(),
|
||||
});
|
||||
}
|
||||
if let (Some(start), Some(end)) =
|
||||
(filter.created_at_gte_millis, filter.created_at_lt_millis)
|
||||
&& start >= end
|
||||
{
|
||||
return Err(Error::InvalidInput {
|
||||
message: "created_at_gte_millis must be less than created_at_lt_millis".into(),
|
||||
});
|
||||
}
|
||||
let mut body = serde_json::json!({"database": self.db_name});
|
||||
if let Some(table) = filter.table {
|
||||
validate_table_name(&table)?;
|
||||
validate_namespace(&filter.namespace_path)?;
|
||||
body["table_ref"] = serde_json::json!({
|
||||
"db_name": self.db_name,
|
||||
"table_id": {"namespace_path": filter.namespace_path, "table_name": table},
|
||||
});
|
||||
}
|
||||
if let Some(state) = filter.state {
|
||||
body["state"] = serde_json::json!(state);
|
||||
}
|
||||
if !filter.job_types.is_empty() {
|
||||
body["job_types"] = serde_json::json!(filter.job_types);
|
||||
}
|
||||
if let Some(start) = filter.created_at_gte_millis {
|
||||
body["created_at_gte_millis"] = start.into();
|
||||
}
|
||||
if let Some(end) = filter.created_at_lt_millis {
|
||||
body["created_at_lt_millis"] = end.into();
|
||||
}
|
||||
if let Some(token) = options.page_token {
|
||||
body["page_token"] = token.into();
|
||||
}
|
||||
@@ -1999,7 +2047,7 @@ mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use crate::connection::ConnectBuilder;
|
||||
use crate::database::Database;
|
||||
use crate::database::{Database, JobListFilter, JobListState};
|
||||
use crate::error::{JobFailure, RemoteErrorMetadata, RemoteResourceIdentity};
|
||||
use crate::graph::{EdgeTable, Endpoint, NodeTable, PropertyGraphDefinition};
|
||||
use crate::materialized_view::CreateMaterializedViewRequest;
|
||||
@@ -4133,7 +4181,10 @@ mod tests {
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(body, serde_json::json!({"page_token": token, "limit": 2}));
|
||||
assert_eq!(
|
||||
body,
|
||||
serde_json::json!({"database": "default", "page_token": token, "limit": 2})
|
||||
);
|
||||
} else {
|
||||
let ns = "team";
|
||||
assert_eq!(
|
||||
@@ -4336,6 +4387,84 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_jobs_filters_every_page_without_eager_fetching() {
|
||||
let count = Arc::new(AtomicUsize::new(0));
|
||||
let seen = count.clone();
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
let page = seen.fetch_add(1, Ordering::SeqCst);
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
let mut expected = serde_json::json!({
|
||||
"database": "default",
|
||||
"table_ref": {
|
||||
"db_name": "default",
|
||||
"table_id": {"namespace_path": ["team", "project"], "table_name": "events"},
|
||||
},
|
||||
"state": "in_progress",
|
||||
"job_types": ["column_refresh", "materialized_view_refresh"],
|
||||
"created_at_gte_millis": 100,
|
||||
"created_at_lt_millis": 200,
|
||||
"limit": 1,
|
||||
});
|
||||
if page > 0 {
|
||||
expected["page_token"] = "next".into();
|
||||
}
|
||||
assert_eq!(body, expected);
|
||||
http::Response::builder().status(200).body(serde_json::json!({
|
||||
"jobs": [{"job_id": format!("job-{page}"), "table": "events", "job_type": "column_refresh", "state": "in_progress", "created_at_millis": 100}],
|
||||
"page_token": (page == 0).then_some("next"),
|
||||
}).to_string()).unwrap()
|
||||
});
|
||||
let filter = JobListFilter {
|
||||
table: Some("events".into()),
|
||||
namespace_path: vec!["team".into(), "project".into()],
|
||||
state: Some(JobListState::InProgress),
|
||||
job_types: vec!["column_refresh".into(), "materialized_view_refresh".into()],
|
||||
created_at_gte_millis: Some(100),
|
||||
created_at_lt_millis: Some(200),
|
||||
};
|
||||
let mut jobs =
|
||||
conn.list_jobs_filtered(filter.clone(), ListingOptions::default().page_limit(1));
|
||||
assert_eq!(count.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(jobs.try_next().await.unwrap().unwrap().job_id, "job-0");
|
||||
assert_eq!(count.load(Ordering::SeqCst), 1);
|
||||
let resumed = conn.list_jobs_filtered(
|
||||
filter,
|
||||
ListingOptions::default()
|
||||
.page_limit(1)
|
||||
.page_token(jobs.page_token().unwrap()),
|
||||
);
|
||||
let remaining = resumed.try_collect::<Vec<_>>().await.unwrap();
|
||||
assert_eq!(remaining.len(), 1);
|
||||
assert_eq!(remaining[0].job_id, "job-1");
|
||||
assert_eq!(count.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_jobs_invalid_filters_never_reach_server() {
|
||||
let conn = Connection::new_with_handler(|_| -> http::Response<String> {
|
||||
panic!("invalid filters must not reach the server")
|
||||
});
|
||||
let namespace_only = JobListFilter {
|
||||
namespace_path: vec!["team".into()],
|
||||
..Default::default()
|
||||
};
|
||||
let invalid_time = JobListFilter {
|
||||
created_at_gte_millis: Some(200),
|
||||
created_at_lt_millis: Some(100),
|
||||
..Default::default()
|
||||
};
|
||||
for filter in [namespace_only, invalid_time] {
|
||||
let mut jobs = conn.list_jobs_filtered(filter, ListingOptions::default());
|
||||
assert!(matches!(
|
||||
jobs.try_next().await,
|
||||
Err(Error::InvalidInput { .. })
|
||||
));
|
||||
assert!(jobs.try_next().await.unwrap().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_jobs_does_not_truncate_after_100_pages() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
|
||||
Reference in New Issue
Block a user