feat: expose server-side job listing filters (#5041)

Expose server-side job filters and lazy Listing indexing, slicing, and length helpers. Document cache, cursor, and SDK/server compatibility behavior.
This commit is contained in:
Jack Ye
2026-11-15 00:50:58 -08:00
committed by GitHub
parent 530d379fc4
commit f17a07a2cd
11 changed files with 756 additions and 53 deletions
+13 -1
View File
@@ -300,7 +300,19 @@ class Connection(object):
page_limit: Optional[int] = None,
) -> NameListing: ...
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> JobListing: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def pause_job(self, job_id: str) -> str: ...
+119 -17
View File
@@ -1602,13 +1602,39 @@ class DBConnection(EnforceOverrides):
)
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = jobs[:5]
"""
raise NotImplementedError(
_append_doc_url(
@@ -2799,16 +2825,52 @@ class LanceDBConnection(DBConnection):
@override
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = jobs[:5]
"""
return Listing(
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
self._conn.list_jobs(
page_token=page_token,
page_limit=page_limit,
table=table,
namespace_path=namespace_path,
state=state,
job_type=job_type,
job_types=job_types,
created_at_gte_millis=created_at_gte_millis,
created_at_lt_millis=created_at_lt_millis,
)
)
@override
@@ -4375,16 +4437,56 @@ class AsyncConnection(object):
)
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> AsyncListing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = []
async for job in jobs:
first_five.append(job)
if len(first_five) == 5:
break
"""
return AsyncListing(
self._inner.list_jobs(page_token=page_token, page_limit=page_limit)
self._inner.list_jobs(
page_token=page_token,
page_limit=page_limit,
table=table,
namespace_path=namespace_path,
state=state,
job_type=job_type,
job_types=job_types,
created_at_gte_millis=created_at_gte_millis,
created_at_lt_millis=created_at_lt_millis,
)
)
async def cancel_job(self, job_id: str) -> bool:
+70 -2
View File
@@ -6,7 +6,19 @@
from __future__ import annotations
import asyncio
from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
from collections import deque
from itertools import islice
from operator import index
from typing import (
Any,
AsyncIterator,
Callable,
Generic,
Iterator,
Optional,
TypeVar,
overload,
)
from .background_loop import LOOP
@@ -59,29 +71,85 @@ class Listing(Iterator[T], Generic[T]):
print(name)
Use ``list(connection.list_views())`` to collect all results.
Indexes and slices refer to the remaining items and do not consume them.
A bounded forward slice, such as ``listing[:5]``, fetches only enough pages
to reach its stop. ``len(listing)``, negative indexes, and open-ended or
reverse slices fetch and cache all remaining results. Iteration then drains
that cache. These operations may perform requests and raise request errors.
"""
def __init__(self, inner: AsyncListing[T]):
self._inner = inner
self._buffer: deque[T] = deque()
def __iter__(self) -> Listing[T]:
return self
def __next__(self) -> T:
if self._buffer:
return self._buffer.popleft()
try:
return LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
raise StopIteration from None
def _fill(self, count: Optional[int]) -> None:
while count is None or len(self._buffer) < count:
try:
item = LOOP.run(self._inner.__anext__())
except StopAsyncIteration:
return
self._buffer.append(item)
def __len__(self) -> int:
"""Fetch and cache every remaining item, returning the remaining count."""
self._fill(None)
return len(self._buffer)
def __bool__(self) -> bool:
self._fill(1)
return bool(self._buffer)
@overload
def __getitem__(self, key: int) -> T: ...
@overload
def __getitem__(self, key: slice) -> list[T]: ...
def __getitem__(self, key: int | slice) -> T | list[T]:
"""Read remaining items without consuming them; bounded slices stay lazy."""
if isinstance(key, slice):
start = 0 if key.start is None else index(key.start)
stop = None if key.stop is None else index(key.stop)
step = 1 if key.step is None else index(key.step)
if step == 0:
raise ValueError("slice step cannot be zero")
if start >= 0 and stop is not None and stop >= 0 and step > 0:
if start >= stop:
return []
self._fill(stop)
start, stop, step = key.indices(len(self._buffer))
return list(
islice(self._buffer, start, stop, min(step, len(self._buffer) + 1))
)
self._fill(None)
return list(self._buffer)[key]
position = index(key)
self._fill(None if position < 0 else position + 1)
return self._buffer[position]
def num_page_results(self) -> int:
"""Return the number of cached items available without another request."""
return self._inner.num_page_results()
return len(self._buffer) + self._inner.num_page_results()
def page_token(self) -> Optional[str]:
"""Return the next request's token.
See [AsyncListing.page_token][lancedb.listing.AsyncListing.page_token]
for initial, cached, final-page, and error behavior.
Indexing, slicing, and counting can also advance this token; drain all
``num_page_results()`` cached items before resuming from it.
"""
return self._inner.page_token()
+42 -6
View File
@@ -1324,16 +1324,52 @@ class RemoteDBConnection(DBConnection):
@override
def list_jobs(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
self,
*,
page_token: Optional[str] = None,
page_limit: Optional[int] = None,
table: Optional[str] = None,
namespace_path: Optional[List[str]] = None,
state: Optional[
Literal["in_progress", "succeeded", "failed", "canceled", "timed_out"]
] = None,
job_type: Optional[str] = None,
job_types: Optional[List[str]] = None,
created_at_gte_millis: Optional[int] = None,
created_at_lt_millis: Optional[int] = None,
) -> Listing[JobInfo]:
"""List server-side jobs across the database's tables.
"""List server-side jobs, optionally filtered before pagination.
Returns a lazy iterator. ``page_limit`` limits each request, not the total;
``page_token`` resumes from a saved token. Requests and errors occur during
iteration. See [pagination state][lancedb.listing.AsyncListing].
``table`` names a table in ``namespace_path`` (the root by default).
Listings include all branches of the table. A namespace filter requires
``table``.
``state`` uses registry states such as ``in_progress``, ``succeeded``,
``failed``, ``timed_out``, and ``canceled``, rather than the SDK's
``running``/``finished`` labels. Set either ``job_type`` or ``job_types``.
Creation-time bounds are Unix milliseconds, inclusive at
``created_at_gte_millis`` and exclusive at ``created_at_lt_millis``.
Returns a lazy iterator. ``page_limit`` limits each request, not the total.
Resume a ``page_token`` with the same filters. Requests and errors occur
during iteration. See [pagination state][lancedb.listing.AsyncListing].
For example, to inspect up to five jobs without fetching all history::
jobs = db.list_jobs(table="events", page_limit=5)
first_five = jobs[:5]
"""
return Listing(
self._conn.list_jobs(page_token=page_token, page_limit=page_limit)
self._conn.list_jobs(
page_token=page_token,
page_limit=page_limit,
table=table,
namespace_path=namespace_path,
state=state,
job_type=job_type,
job_types=job_types,
created_at_gte_millis=created_at_gte_millis,
created_at_lt_millis=created_at_lt_millis,
)
)
@override
+177 -1
View File
@@ -268,7 +268,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous)
if kind == "jobs":
assert verb == "POST"
assert path.path == "/v1/jobs/list"
assert body == {"page_token": token, "limit": 2}
assert body == {"database": "dev", "page_token": token, "limit": 2}
else:
namespace = "team"
resource = kind.removesuffix("s")
@@ -326,3 +326,179 @@ async def test_listing_invalid_limit_does_not_fetch(listing_server, kind, page_l
with pytest.raises(ValueError, match="limit"):
await items.__anext__()
assert requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("namespace", [[], ["team", "project"]])
@pytest.mark.parametrize("multiple_types", [False, True])
async def test_job_filters_persist_across_pages_and_resume(
listing_server, asynchronous, namespace, multiple_types
):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page("jobs", names, token))
for names, token in [(["a"], "empty"), ([], "last"), (["b"], None)]
)
db = await connection(endpoint, asynchronous)
types = ["column_refresh", "materialized_view_refresh"]
filters = dict(
table="events",
namespace_path=namespace,
state="in_progress",
created_at_gte_millis=100,
created_at_lt_millis=200,
**({"job_types": types} if multiple_types else {"job_type": types[0]}),
)
items = db.list_jobs(page_limit=1, **filters)
assert requests == []
assert (await advance(items, asynchronous)).job_id == "a"
assert len(requests) == 1
resumed = db.list_jobs(page_token=items.page_token(), page_limit=1, **filters)
assert (await advance(resumed, asynchronous)).job_id == "b"
assert resumed.page_token() is None
assert len(requests) == 3
expected = dict(
database="dev",
limit=1,
table_ref={
"db_name": "dev",
"table_id": {"namespace_path": namespace, "table_name": "events"},
},
state="in_progress",
job_types=types if multiple_types else types[:1],
created_at_gte_millis=100,
created_at_lt_millis=200,
)
for (_, path, body), token in zip(requests, [None, "empty", "last"]):
assert path == "/v1/jobs/list"
assert body == {**expected, **({"page_token": token} if token else {})}
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
"filters",
[
{},
{"state": "failed"},
{"job_types": ["column_refresh"]},
{"created_at_lt_millis": 200},
],
)
async def test_job_listing_without_table_stays_in_connected_database(
listing_server, asynchronous, filters
):
endpoint, requests, responses = listing_server
responses.append((200, response_page("jobs", ["a"], None)))
db = await connection(endpoint, asynchronous)
items = db.list_jobs(**filters)
assert (await advance(items, asynchronous)).job_id == "a"
assert requests[0][2] == {"database": "dev", **filters}
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
"filters, message",
[
({"namespace_path": ["team"]}, "requires a table"),
({"created_at_gte_millis": 200, "created_at_lt_millis": 200}, "less than"),
({"created_at_gte_millis": 201, "created_at_lt_millis": 200}, "less than"),
({"job_type": "column_refresh", "job_types": []}, "only one"),
({"state": "unknown"}, "state must be"),
],
)
async def test_invalid_job_filters_do_not_fetch(
listing_server, asynchronous, filters, message
):
endpoint, requests, _ = listing_server
db = await connection(endpoint, asynchronous)
with pytest.raises(ValueError, match=message):
items = db.list_jobs(**filters)
await advance(items, asynchronous)
assert requests == []
@pytest.mark.asyncio
async def test_listing_slice_and_index_fetch_only_needed_pages(listing_server):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page("jobs", names, token))
for names, token in [(["a", "b"], "second"), (["c", "d"], "last")]
)
db = await connection(endpoint, False)
items = db.list_jobs(table="events", page_limit=2)
assert items[:0] == []
assert items[5:2] == []
assert requests == []
assert [job.job_id for job in items[:2]] == ["a", "b"]
assert len(requests) == 1
assert items.num_page_results() == 2
assert items.page_token() == "second"
assert items[1].job_id == "b"
assert next(items).job_id == "a"
assert [job.job_id for job in items[:2]] == ["b", "c"]
assert len(requests) == 2
assert items.num_page_results() == 3
assert items.page_token() == "last"
assert [job.job_id for job in items[0:3:2][:1]] == ["b"]
assert len(requests) == 2
@pytest.mark.asyncio
async def test_listing_length_and_slices_preserve_remaining_items(listing_server):
endpoint, requests, responses = listing_server
responses.extend(
(200, response_page("jobs", names, token))
for names, token in [(["a", "b"], "empty"), ([], "last"), (["c", "d"], None)]
)
db = await connection(endpoint, False)
items = db.list_jobs(page_limit=2)
assert bool(items)
assert len(requests) == 1
assert next(items).job_id == "a"
assert len(items) == 3
assert len(requests) == 3
assert items.num_page_results() == 3
assert items.page_token() is None
assert items[-1].job_id == "d"
assert [job.job_id for job in items[::-1]] == ["d", "c", "b"]
assert [job.job_id for job in items[1:]] == ["c", "d"]
assert [job.job_id for job in items[:100:2]] == ["b", "d"]
assert [job.job_id for job in items[: 2**100 : 2**100]] == ["b"]
with pytest.raises(IndexError):
_ = items[3]
assert [job.job_id for job in items] == ["b", "c", "d"]
assert len(items) == 0
assert not items
assert len(requests) == 3
@pytest.mark.asyncio
async def test_listing_invalid_slice_does_not_fetch(listing_server):
endpoint, requests, _ = listing_server
db = await connection(endpoint, False)
items = db.list_jobs()
with pytest.raises(ValueError, match="step"):
_ = items[::0]
with pytest.raises(TypeError):
_ = items["invalid"]
with pytest.raises(TypeError):
_ = items[:1.5]
assert requests == []
@pytest.mark.asyncio
async def test_listing_slice_failure_preserves_cache_and_retry_token(listing_server):
endpoint, requests, responses = listing_server
responses.extend([(200, response_page("jobs", ["a", "b"], "retry")), (401, {})])
db = await connection(endpoint, False)
items = db.list_jobs(page_limit=2)
with pytest.raises(HttpError):
_ = items[:3]
assert items.page_token() == "retry"
assert items.num_page_results() == 2
assert [job.job_id for job in items[:2]] == ["a", "b"]
assert [job.job_id for job in items] == ["a", "b"]
assert len(requests) == 2
+35 -3
View File
@@ -23,7 +23,7 @@ use lancedb::{
connection::Connection as LanceConnection,
connection::NamespaceClientPushdownOperation,
database::namespace::LanceNamespaceDatabase,
database::{CreateTableMode, Database, ReadConsistency},
database::{CreateTableMode, Database, JobListFilter, JobListState, ReadConsistency},
function::{GpuClass as LanceGpuClass, RefreshColumnResult},
graph::PropertyGraphDefinition,
materialized_view::CreateMaterializedViewBuilder,
@@ -1319,15 +1319,47 @@ impl Connection {
))
}
#[pyo3(signature = (*, page_token=None, page_limit=None))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (*, page_token=None, page_limit=None, table=None, namespace_path=None, state=None, job_type=None, job_types=None, created_at_gte_millis=None, created_at_lt_millis=None))]
pub fn list_jobs(
&self,
page_token: Option<String>,
page_limit: Option<u32>,
table: Option<String>,
namespace_path: Option<Vec<String>>,
state: Option<String>,
job_type: Option<String>,
job_types: Option<Vec<String>>,
created_at_gte_millis: Option<i64>,
created_at_lt_millis: Option<i64>,
) -> PyResult<crate::listing::JobListing> {
if job_type.is_some() && job_types.is_some() {
return Err(PyValueError::new_err(
"set only one of job_type and job_types",
));
}
let mut filter = JobListFilter::default();
filter.table = table;
filter.namespace_path = namespace_path.unwrap_or_default();
filter.state = state
.as_deref()
.map(|state| match state {
"in_progress" => Ok(JobListState::InProgress),
"succeeded" => Ok(JobListState::Succeeded),
"failed" => Ok(JobListState::Failed),
"canceled" => Ok(JobListState::Canceled),
"timed_out" => Ok(JobListState::TimedOut),
_ => Err(PyValueError::new_err(
"state must be one of: in_progress, succeeded, failed, canceled, timed_out",
)),
})
.transpose()?;
filter.job_types = job_types.unwrap_or_else(|| job_type.into_iter().collect());
filter.created_at_gte_millis = created_at_gte_millis;
filter.created_at_lt_millis = created_at_lt_millis;
Ok(crate::listing::JobListing::new(
self.get_inner()?
.list_jobs(crate::listing::options(page_token, page_limit)),
.list_jobs_filtered(filter, crate::listing::options(page_token, page_limit)),
))
}