mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-07 22:08:56 +00:00
feat!: replace get_job/job_history with describe_job/query_job_events (#4130)
A 1M-row column refresh over 200 fragments produced no visible result,
and the client could only ever say `"running"`. Everything needed to
diagnose it already existed server-side — the job registry records a
`claim`/`claim_complete` pair per fragment carrying `rows_processed` —
but none of it was reachable.
## Before
Four ways to ask about a job, none of which told you much.
```python
job = table.refresh_column_async("embedding")
job.status() # "running". That was the entire debug surface.
db.get_job(job_id) # state, and a spec. No result, no progress.
db.job_history(job_id) # raw record batches, no limit, no filter
db.job(job_id) # a handle that knew nothing
```
## After
Open a job the way you open a table; the handle answers everything.
```python
job = db.open_job(job_id) # raises JobNotFoundError if there is no such job
```
```python
>>> print(job)
Job(
id='job-1',
state='failed',
job_type='refresh_column',
creation_ms=1757000000000,
spec={
"column": "embedding",
"num_workers": 4
},
failure=JobFailureInfo(phase='execute', message='worker died', retryable=True),
)
```
Individual fields are there too — `job.state`, `job.job_type`,
`job.creation_ms`, `job.spec`, `job.result`, `job.failure` — and
`job.result` carries `rows_assigned` / `rows_failed` as soon as the job
succeeds, with no `wait()` required.
Per-fragment progress *while it is still running*:
```python
done = job.events(filter="state = 'claim_complete'", limit=10_000)
done.column("rows_processed").to_pylist() # [5000, 5000, ...]
```
The handle an async action returns is the same object, one `refresh()`
away:
```python
job = table.refresh_column_async("embedding")
job.refresh()
job.state, job.result
```
TypeScript is the same experience, down to `console.log`:
```ts
const job = await db.openJob(jobId); // rejects if there is no such job
console.log(job); // same multi-line layout
job.state; job.jobType; job.spec; job.result; job.failure;
const done = await job.events({ filter: "state = 'claim_complete'", limit: 10_000 });
```
## Why each piece matters
- **A result without waiting.** `rows_assigned` / `rows_failed` used to
live only on the terminal result, so a job that never terminated
reported nothing at all.
- **`limit`.** The server caps event rows at 1000 and truncates without
saying so, which silently hid most of a 200-fragment job's history.
- **`filter`.** `claim_complete` rows carry per-claim `rows_processed` —
the only progress signal that exists mid-flight.
- **Events outlive the worker.** They live in the job registry, not in
pod logs that vanish with the pod.
- **One place to ask.** `open_job` replaces `describe_job`,
`query_job_events` and `job`, so a question about a job has one answer
instead of one per calling location.
- **A missing job is an error, not a `None`.** The common case is a job
id copied out of a log, where absence is the surprise worth raising —
and it matches `open_table`.
- **Printing is the debug surface.** Every field on its own line, JSON
payloads keeping their structure. An unrefreshed handle stays on one
line, because there is nothing to lay out.
- **In-process jobs say so.** A local refresh reports `state` and leaves
the rest null rather than inventing fields it has no record for.
`list_jobs` and `cancel_job` stay as they were: one lists, the other is
a one-shot action that should not need a describe first.
## Breaking
All shipped in 0.38.0. No deprecated aliases.
| Was | Now |
| --- | --- |
| `Connection.get_job` → `describe_job` | `Connection.open_job` returns
a populated `Job`, or raises |
| `Connection.job_history` → `query_job_events` | `job.events(...)` |
| `Connection.job` | `Connection.open_job` |
| Python events → `List[pa.RecordBatch]` | `pa.Table` |
| `JobDescription.spec_json` / `.result_json` | internal; use `job.spec`
/ `job.result` |
Node's `Job` is now a TypeScript class wrapping the native handle, so it
returns an Arrow table and parsed values like Python does. New
`Error::JobNotFound` / `JobNotFoundError`; the three job exceptions are
now in the Python API reference.
This commit is contained in:
Jack Ye
@@ -939,6 +939,7 @@ describe("remote connection jobs surface", () => {
|
||||
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
|
||||
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
|
||||
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
|
||||
const queryEventsPayloads: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
@@ -967,6 +968,16 @@ describe("remote connection jobs surface", () => {
|
||||
);
|
||||
}
|
||||
} else if (req.url === "/v1/jobs/describe") {
|
||||
if (payload["job_id"] === "job-2") {
|
||||
res
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end(
|
||||
'{"job_id": "job-2", "job_type": "refresh_column", ' +
|
||||
'"job_state": "DONE", "creation_ms": 2000, ' +
|
||||
'"result": {"rows_assigned": 1000000}}',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (payload["job_id"] !== "job-1") {
|
||||
res.writeHead(404).end("no such job");
|
||||
return;
|
||||
@@ -988,6 +999,7 @@ describe("remote connection jobs surface", () => {
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end('{"job_id": "job-1"}');
|
||||
} else if (req.url === "/v1/jobs/query_events") {
|
||||
queryEventsPayloads.push(payload);
|
||||
res
|
||||
.writeHead(200, {
|
||||
"Content-Type": "application/vnd.apache.arrow.stream",
|
||||
@@ -1004,22 +1016,65 @@ describe("remote connection jobs surface", () => {
|
||||
expect(jobs[0].state).toEqual("running");
|
||||
expect(jobs[1].state).toEqual("finished");
|
||||
|
||||
const description = await db.getJob("job-1");
|
||||
expect(description?.state).toEqual("failed");
|
||||
expect(JSON.parse(description?.specJson ?? "")).toEqual({
|
||||
column: "vec",
|
||||
});
|
||||
expect(description?.failure?.message).toEqual("worker died");
|
||||
expect(await db.getJob("missing")).toBeNull();
|
||||
|
||||
expect(await db.cancelJob("job-1")).toBe(true);
|
||||
expect(await db.cancelJob("missing")).toBe(false);
|
||||
|
||||
const history = await db.jobHistory("job-1");
|
||||
expect(history.numRows).toEqual(2);
|
||||
// Opening a job hands back a populated handle; a missing one rejects.
|
||||
await expect(db.openJob("missing")).rejects.toThrow("not found");
|
||||
const finished = await db.openJob("job-2");
|
||||
expect(finished.state).toEqual("finished");
|
||||
expect(finished.result).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
rows_assigned: 1000000,
|
||||
});
|
||||
|
||||
const job = db.job("job-1");
|
||||
const job = await db.openJob("job-1");
|
||||
expect(job.id).toEqual("job-1");
|
||||
|
||||
// openJob already populated the handle; refresh() re-reads it.
|
||||
expect(job.state).toEqual("failed");
|
||||
await job.refresh();
|
||||
expect(job.state).toEqual("failed");
|
||||
expect(job.jobType).toEqual("create_index");
|
||||
expect(job.creationMs).toEqual(1000);
|
||||
expect(job.spec).toEqual({ column: "vec" });
|
||||
expect(job.result).toBeNull();
|
||||
expect(job.failure?.message).toEqual("worker died");
|
||||
|
||||
// The handle reaches its own events, supplying its job id.
|
||||
const jobEvents = await job.events({
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
expect(jobEvents.numRows).toEqual(2);
|
||||
expect(queryEventsPayloads.pop()).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
job_id: "job-1",
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
|
||||
// Printing lays every known field out on its own line, with the JSON
|
||||
// payloads indented rather than crammed onto one line.
|
||||
expect(`${job}`).toEqual(
|
||||
[
|
||||
"Job(",
|
||||
' id="job-1",',
|
||||
' state="failed",',
|
||||
' jobType="create_index",',
|
||||
" creationMs=1000,",
|
||||
" spec={",
|
||||
' "column": "vec"',
|
||||
" },",
|
||||
" failure={",
|
||||
' "phase": "execute",',
|
||||
' "message": "worker died",',
|
||||
' "retryable": true',
|
||||
" },",
|
||||
")",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(await job.status()).toEqual("failed");
|
||||
await expect(job.wait()).rejects.toThrow("worker died");
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user