mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-06 21:39:03 +00:00
feat!: replace get_job/job_history with describe_job/query_job_events (#4130)
A 1M-row column refresh over 200 fragments produced no visible result,
and the client could only ever say `"running"`. Everything needed to
diagnose it already existed server-side — the job registry records a
`claim`/`claim_complete` pair per fragment carrying `rows_processed` —
but none of it was reachable.
## Before
Four ways to ask about a job, none of which told you much.
```python
job = table.refresh_column_async("embedding")
job.status() # "running". That was the entire debug surface.
db.get_job(job_id) # state, and a spec. No result, no progress.
db.job_history(job_id) # raw record batches, no limit, no filter
db.job(job_id) # a handle that knew nothing
```
## After
Open a job the way you open a table; the handle answers everything.
```python
job = db.open_job(job_id) # raises JobNotFoundError if there is no such job
```
```python
>>> print(job)
Job(
id='job-1',
state='failed',
job_type='refresh_column',
creation_ms=1757000000000,
spec={
"column": "embedding",
"num_workers": 4
},
failure=JobFailureInfo(phase='execute', message='worker died', retryable=True),
)
```
Individual fields are there too — `job.state`, `job.job_type`,
`job.creation_ms`, `job.spec`, `job.result`, `job.failure` — and
`job.result` carries `rows_assigned` / `rows_failed` as soon as the job
succeeds, with no `wait()` required.
Per-fragment progress *while it is still running*:
```python
done = job.events(filter="state = 'claim_complete'", limit=10_000)
done.column("rows_processed").to_pylist() # [5000, 5000, ...]
```
The handle an async action returns is the same object, one `refresh()`
away:
```python
job = table.refresh_column_async("embedding")
job.refresh()
job.state, job.result
```
TypeScript is the same experience, down to `console.log`:
```ts
const job = await db.openJob(jobId); // rejects if there is no such job
console.log(job); // same multi-line layout
job.state; job.jobType; job.spec; job.result; job.failure;
const done = await job.events({ filter: "state = 'claim_complete'", limit: 10_000 });
```
## Why each piece matters
- **A result without waiting.** `rows_assigned` / `rows_failed` used to
live only on the terminal result, so a job that never terminated
reported nothing at all.
- **`limit`.** The server caps event rows at 1000 and truncates without
saying so, which silently hid most of a 200-fragment job's history.
- **`filter`.** `claim_complete` rows carry per-claim `rows_processed` —
the only progress signal that exists mid-flight.
- **Events outlive the worker.** They live in the job registry, not in
pod logs that vanish with the pod.
- **One place to ask.** `open_job` replaces `describe_job`,
`query_job_events` and `job`, so a question about a job has one answer
instead of one per calling location.
- **A missing job is an error, not a `None`.** The common case is a job
id copied out of a log, where absence is the surprise worth raising —
and it matches `open_table`.
- **Printing is the debug surface.** Every field on its own line, JSON
payloads keeping their structure. An unrefreshed handle stays on one
line, because there is nothing to lay out.
- **In-process jobs say so.** A local refresh reports `state` and leaves
the rest null rather than inventing fields it has no record for.
`list_jobs` and `cancel_job` stay as they were: one lists, the other is
a one-shot action that should not need a describe first.
## Breaking
All shipped in 0.38.0. No deprecated aliases.
| Was | Now |
| --- | --- |
| `Connection.get_job` → `describe_job` | `Connection.open_job` returns
a populated `Job`, or raises |
| `Connection.job_history` → `query_job_events` | `job.events(...)` |
| `Connection.job` | `Connection.open_job` |
| Python events → `List[pa.RecordBatch]` | `pa.Table` |
| `JobDescription.spec_json` / `.result_json` | internal; use `job.spec`
/ `job.result` |
Node's `Job` is now a TypeScript class wrapping the native handle, so it
returns an Arrow table and parsed values like Python does. New
`Error::JobNotFound` / `JobNotFoundError`; the three job exceptions are
now in the Python API reference.
This commit is contained in:
@@ -939,6 +939,7 @@ describe("remote connection jobs surface", () => {
|
||||
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
|
||||
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
|
||||
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
|
||||
const queryEventsPayloads: Record<string, unknown>[] = [];
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
@@ -967,6 +968,16 @@ describe("remote connection jobs surface", () => {
|
||||
);
|
||||
}
|
||||
} else if (req.url === "/v1/jobs/describe") {
|
||||
if (payload["job_id"] === "job-2") {
|
||||
res
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end(
|
||||
'{"job_id": "job-2", "job_type": "refresh_column", ' +
|
||||
'"job_state": "DONE", "creation_ms": 2000, ' +
|
||||
'"result": {"rows_assigned": 1000000}}',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (payload["job_id"] !== "job-1") {
|
||||
res.writeHead(404).end("no such job");
|
||||
return;
|
||||
@@ -988,6 +999,7 @@ describe("remote connection jobs surface", () => {
|
||||
.writeHead(200, { "Content-Type": "application/json" })
|
||||
.end('{"job_id": "job-1"}');
|
||||
} else if (req.url === "/v1/jobs/query_events") {
|
||||
queryEventsPayloads.push(payload);
|
||||
res
|
||||
.writeHead(200, {
|
||||
"Content-Type": "application/vnd.apache.arrow.stream",
|
||||
@@ -1004,22 +1016,65 @@ describe("remote connection jobs surface", () => {
|
||||
expect(jobs[0].state).toEqual("running");
|
||||
expect(jobs[1].state).toEqual("finished");
|
||||
|
||||
const description = await db.getJob("job-1");
|
||||
expect(description?.state).toEqual("failed");
|
||||
expect(JSON.parse(description?.specJson ?? "")).toEqual({
|
||||
column: "vec",
|
||||
});
|
||||
expect(description?.failure?.message).toEqual("worker died");
|
||||
expect(await db.getJob("missing")).toBeNull();
|
||||
|
||||
expect(await db.cancelJob("job-1")).toBe(true);
|
||||
expect(await db.cancelJob("missing")).toBe(false);
|
||||
|
||||
const history = await db.jobHistory("job-1");
|
||||
expect(history.numRows).toEqual(2);
|
||||
// Opening a job hands back a populated handle; a missing one rejects.
|
||||
await expect(db.openJob("missing")).rejects.toThrow("not found");
|
||||
const finished = await db.openJob("job-2");
|
||||
expect(finished.state).toEqual("finished");
|
||||
expect(finished.result).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
rows_assigned: 1000000,
|
||||
});
|
||||
|
||||
const job = db.job("job-1");
|
||||
const job = await db.openJob("job-1");
|
||||
expect(job.id).toEqual("job-1");
|
||||
|
||||
// openJob already populated the handle; refresh() re-reads it.
|
||||
expect(job.state).toEqual("failed");
|
||||
await job.refresh();
|
||||
expect(job.state).toEqual("failed");
|
||||
expect(job.jobType).toEqual("create_index");
|
||||
expect(job.creationMs).toEqual(1000);
|
||||
expect(job.spec).toEqual({ column: "vec" });
|
||||
expect(job.result).toBeNull();
|
||||
expect(job.failure?.message).toEqual("worker died");
|
||||
|
||||
// The handle reaches its own events, supplying its job id.
|
||||
const jobEvents = await job.events({
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
expect(jobEvents.numRows).toEqual(2);
|
||||
expect(queryEventsPayloads.pop()).toEqual({
|
||||
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
|
||||
job_id: "job-1",
|
||||
limit: 500,
|
||||
filter: "state = 'claim_complete'",
|
||||
});
|
||||
|
||||
// Printing lays every known field out on its own line, with the JSON
|
||||
// payloads indented rather than crammed onto one line.
|
||||
expect(`${job}`).toEqual(
|
||||
[
|
||||
"Job(",
|
||||
' id="job-1",',
|
||||
' state="failed",',
|
||||
' jobType="create_index",',
|
||||
" creationMs=1000,",
|
||||
" spec={",
|
||||
' "column": "vec"',
|
||||
" },",
|
||||
" failure={",
|
||||
' "phase": "execute",',
|
||||
' "message": "worker died",',
|
||||
' "retryable": true',
|
||||
" },",
|
||||
")",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
expect(await job.status()).toEqual("failed");
|
||||
await expect(job.wait()).rejects.toThrow("worker died");
|
||||
},
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { tableFromIPC } from "apache-arrow";
|
||||
import {
|
||||
Data,
|
||||
SchemaLike,
|
||||
@@ -16,6 +15,7 @@ import {
|
||||
makeEmptyTable,
|
||||
} from "./arrow";
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { Job } from "./job";
|
||||
import {
|
||||
MaterializedView,
|
||||
MaterializedViewSelect,
|
||||
@@ -27,8 +27,6 @@ import type {
|
||||
CreateNamespaceResponse,
|
||||
DescribeNamespaceResponse,
|
||||
DropNamespaceResponse,
|
||||
Job,
|
||||
JobDescription,
|
||||
JobInfo,
|
||||
ListNamespacesResponse,
|
||||
ListTablesResponse,
|
||||
@@ -557,24 +555,19 @@ export abstract class Connection {
|
||||
): Promise<void>;
|
||||
|
||||
/**
|
||||
* A {@link Job} handle for a server-side job by id.
|
||||
* Open a server-side job by id, returning a handle with its record already
|
||||
* populated. Rejects when the server has no such job, the way
|
||||
* {@link Connection.openTable} does for a missing table.
|
||||
*
|
||||
* The handle is constructed without a server round trip; an unknown id
|
||||
* surfaces when the handle is used. Dropping the handle has no effect on
|
||||
* the job itself.
|
||||
* The returned {@link Job} answers for its own state, specification,
|
||||
* result, failure and event history, so there is no separate
|
||||
* connection-level call for any of them.
|
||||
*/
|
||||
abstract job(jobId: string): Job;
|
||||
abstract openJob(jobId: string): Promise<Job>;
|
||||
|
||||
/** List server-side jobs across the database's tables. */
|
||||
abstract listJobs(): Promise<JobInfo[]>;
|
||||
|
||||
/**
|
||||
* Describe a single server-side job by id.
|
||||
*
|
||||
* Resolves to `null` when the server has no such job.
|
||||
*/
|
||||
abstract getJob(jobId: string): Promise<JobDescription | null>;
|
||||
|
||||
/**
|
||||
* Request cancellation of a server-side job by id.
|
||||
*
|
||||
@@ -582,13 +575,6 @@ export abstract class Connection {
|
||||
* such job exists. Cancelling an already-terminal job is a no-op success.
|
||||
*/
|
||||
abstract cancelJob(jobId: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* The lifecycle event history of a server-side job, as an Arrow table.
|
||||
*
|
||||
* Lists history across all jobs when `jobId` is omitted.
|
||||
*/
|
||||
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
|
||||
}
|
||||
|
||||
/** @hideconstructor */
|
||||
@@ -869,7 +855,7 @@ export class LocalConnection extends Connection {
|
||||
}
|
||||
|
||||
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
|
||||
return this.inner.dropTableAsync(name, namespacePath ?? []);
|
||||
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
|
||||
}
|
||||
|
||||
async dropAllTables(namespacePath?: string[]): Promise<void> {
|
||||
@@ -928,29 +914,17 @@ export class LocalConnection extends Connection {
|
||||
);
|
||||
}
|
||||
|
||||
job(jobId: string): Job {
|
||||
return this.inner.job(jobId);
|
||||
async openJob(jobId: string): Promise<Job> {
|
||||
return new Job(await this.inner.openJob(jobId));
|
||||
}
|
||||
|
||||
async listJobs(): Promise<JobInfo[]> {
|
||||
return this.inner.listJobs();
|
||||
}
|
||||
|
||||
async getJob(jobId: string): Promise<JobDescription | null> {
|
||||
return this.inner.getJob(jobId);
|
||||
}
|
||||
|
||||
async cancelJob(jobId: string): Promise<boolean> {
|
||||
return this.inner.cancelJob(jobId);
|
||||
}
|
||||
|
||||
async jobHistory(jobId?: string): Promise<ArrowTable> {
|
||||
const buf = await this.inner.jobHistory(jobId);
|
||||
if (buf.length === 0) {
|
||||
return new ArrowTable();
|
||||
}
|
||||
return tableFromIPC(buf);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -94,13 +94,9 @@ export {
|
||||
RenameTableOptions,
|
||||
} from "./connection";
|
||||
|
||||
export {
|
||||
Job,
|
||||
JobDescription,
|
||||
JobFailureInfo,
|
||||
JobInfo,
|
||||
Session,
|
||||
} from "./native.js";
|
||||
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
||||
|
||||
export { Job, JobEventsOptions } from "./job";
|
||||
|
||||
export {
|
||||
AutoQuery,
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
|
||||
import { JobFailureInfo, Job as NativeJob } from "./native";
|
||||
|
||||
/** Which of a job's events {@link Job.events} returns. */
|
||||
export interface JobEventsOptions {
|
||||
/** Maximum event rows to return, up to the server maximum of 10,000. */
|
||||
limit?: number;
|
||||
/** SQL-like filter over the event columns. */
|
||||
filter?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A handle to an operation that may still be running.
|
||||
*
|
||||
* The operation may already be complete when the handle is created.
|
||||
*
|
||||
* The detail getters read what the handle last observed. Submitting an
|
||||
* operation returns only a job id, so populating them eagerly would cost an
|
||||
* extra round trip on every call:
|
||||
*
|
||||
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
|
||||
* - {@link Job.wait} records the terminal state it establishes, but not the
|
||||
* rest of the record.
|
||||
* - Everything is null until one of those runs.
|
||||
*
|
||||
* @hideconstructor
|
||||
*/
|
||||
export class Job {
|
||||
private readonly inner: NativeJob;
|
||||
|
||||
constructor(inner: NativeJob) {
|
||||
this.inner = inner;
|
||||
}
|
||||
|
||||
/**
|
||||
* Identifies the operation on the server that is running it.
|
||||
*
|
||||
* Operations that run in this process have no server id. The value is
|
||||
* opaque: parsing it or storing it to resume the job later is not supported.
|
||||
*/
|
||||
get id(): string | null {
|
||||
return this.inner.id ?? null;
|
||||
}
|
||||
|
||||
/** The last observed lifecycle state, without contacting the backend. */
|
||||
get state(): string | null {
|
||||
return this.inner.state ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The job's type, as the server names it. Null for an in-process job, which
|
||||
* has no server-side record.
|
||||
*/
|
||||
get jobType(): string | null {
|
||||
return this.inner.jobType ?? null;
|
||||
}
|
||||
|
||||
/** When the job was created, in milliseconds since the epoch. */
|
||||
get creationMs(): number | null {
|
||||
return this.inner.creationMs ?? null;
|
||||
}
|
||||
|
||||
/** The job-type-specific specification it was submitted with. */
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
get spec(): any | null {
|
||||
return parseJson(this.inner.specJson);
|
||||
}
|
||||
|
||||
/**
|
||||
* The job-type-specific terminal result. Null until the job succeeds, so a
|
||||
* job that never terminates reports its progress through {@link Job.events}
|
||||
* instead.
|
||||
*/
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
get result(): any | null {
|
||||
return parseJson(this.inner.resultJson);
|
||||
}
|
||||
|
||||
/** Why the job failed, when it failed and the server reports a reason. */
|
||||
get failure(): JobFailureInfo | null {
|
||||
return this.inner.failure ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The operation's current lifecycle state: "running", "finished", "failed",
|
||||
* or "cancelled".
|
||||
*
|
||||
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
|
||||
* terminal failure state. Also refreshes the getters above.
|
||||
*/
|
||||
async status(): Promise<string> {
|
||||
return this.inner.status();
|
||||
}
|
||||
|
||||
/** Wait until the operation reaches a terminal state. */
|
||||
async wait(): Promise<void> {
|
||||
return this.inner.wait();
|
||||
}
|
||||
|
||||
/** Request cancellation. Cancelling a finished operation is a no-op. */
|
||||
async cancel(): Promise<void> {
|
||||
return this.inner.cancel();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the backend for this job's current state, and for a server-side job
|
||||
* its full record, then cache it for the getters above.
|
||||
*/
|
||||
async refresh(): Promise<void> {
|
||||
return this.inner.refresh();
|
||||
}
|
||||
|
||||
/**
|
||||
* This job's recorded lifecycle events.
|
||||
*
|
||||
* Where the getters above report a terminal result only once the job reaches
|
||||
* one, events are written as the job runs and outlive the workers that
|
||||
* produced them. A distributed job records a `claim`/`claim_complete` pair
|
||||
* per unit of work, each carrying `rows_processed`, so a job that never
|
||||
* finishes still accounts for what it did.
|
||||
*
|
||||
* The server caps results at 1000 rows by default and 10,000 at most, and
|
||||
* truncates without saying so, so pass `limit` for a job that emits an event
|
||||
* per fragment. `filter` is a SQL-like expression over the `state`,
|
||||
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
||||
*/
|
||||
async events(options?: JobEventsOptions): Promise<ArrowTable> {
|
||||
const buf = await this.inner.events(options?.limit, options?.filter);
|
||||
if (buf.length === 0) {
|
||||
return new ArrowTable();
|
||||
}
|
||||
return tableFromIPC(buf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Every field the handle currently knows, one per line, with the JSON
|
||||
* payloads indented -- a refresh job's spec and result are the point of
|
||||
* printing it.
|
||||
*/
|
||||
toString(): string {
|
||||
if (this.state === null) {
|
||||
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
|
||||
return `Job(${known}not refreshed)`;
|
||||
}
|
||||
const fields: string[] = [];
|
||||
if (this.id !== null) {
|
||||
fields.push(`id=${JSON.stringify(this.id)}`);
|
||||
}
|
||||
fields.push(`state=${JSON.stringify(this.state)}`);
|
||||
if (this.jobType !== null) {
|
||||
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
|
||||
}
|
||||
if (this.creationMs !== null) {
|
||||
fields.push(`creationMs=${this.creationMs}`);
|
||||
}
|
||||
for (const [name, value] of [
|
||||
["spec", this.spec],
|
||||
["result", this.result],
|
||||
] as const) {
|
||||
if (value !== null) {
|
||||
fields.push(`${name}=${indentJson(value)}`);
|
||||
}
|
||||
}
|
||||
if (this.failure !== null) {
|
||||
fields.push(`failure=${indentJson(this.failure)}`);
|
||||
}
|
||||
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
|
||||
}
|
||||
|
||||
[Symbol.for("nodejs.util.inspect.custom")](): string {
|
||||
return this.toString();
|
||||
}
|
||||
}
|
||||
|
||||
const REPR_INDENT = " ";
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
function indentJson(value: any): string {
|
||||
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
|
||||
function parseJson(raw: string | null | undefined): any | null {
|
||||
return raw === null || raw === undefined ? null : JSON.parse(raw);
|
||||
}
|
||||
+11
-9
@@ -19,6 +19,7 @@ import {
|
||||
|
||||
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
|
||||
import { IndexOptions } from "./indices";
|
||||
import { Job } from "./job";
|
||||
import { MergeInsertBuilder } from "./merge";
|
||||
import {
|
||||
AddColumnsResult,
|
||||
@@ -30,7 +31,6 @@ import {
|
||||
DropColumnsResult,
|
||||
IndexConfig,
|
||||
IndexStatistics,
|
||||
Job,
|
||||
LsmStats,
|
||||
Branches as NativeBranches,
|
||||
OptimizeStats,
|
||||
@@ -1124,13 +1124,15 @@ export class LocalTable extends Table {
|
||||
): Promise<Job> {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: skip
|
||||
const nativeIndex = (options?.config as any)?.inner;
|
||||
return await this.inner.createIndexAsync(
|
||||
nativeIndex,
|
||||
column,
|
||||
options?.replace,
|
||||
options?.waitTimeoutSeconds,
|
||||
options?.name,
|
||||
options?.train,
|
||||
return new Job(
|
||||
await this.inner.createIndexAsync(
|
||||
nativeIndex,
|
||||
column,
|
||||
options?.replace,
|
||||
options?.waitTimeoutSeconds,
|
||||
options?.name,
|
||||
options?.train,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1313,7 +1315,7 @@ export class LocalTable extends Table {
|
||||
}
|
||||
|
||||
async refreshColumnAsync(column: string): Promise<Job> {
|
||||
return await this.inner.refreshColumnAsync(column);
|
||||
return new Job(await this.inner.refreshColumnAsync(column));
|
||||
}
|
||||
|
||||
async refreshMaterializedView(
|
||||
|
||||
@@ -442,13 +442,15 @@ impl Connection {
|
||||
self.get_inner()?.drop_all_tables(&ns).await.default_error()
|
||||
}
|
||||
|
||||
/// A `Job` handle for a server-side job by id.
|
||||
/// Open a server-side job by id, returning a handle with its record
|
||||
/// already populated. Rejects when the server has no such job.
|
||||
///
|
||||
/// The handle is constructed without a server round trip; an unknown id
|
||||
/// surfaces when the handle is used.
|
||||
#[napi]
|
||||
pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
|
||||
let job = self.get_inner()?.job(job_id).default_error()?;
|
||||
/// The returned handle answers for its own state, specification, result,
|
||||
/// failure and event history, so there is no separate connection-level
|
||||
/// call for any of them.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
|
||||
let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
|
||||
Ok(crate::job::Job::new(job))
|
||||
}
|
||||
|
||||
@@ -459,17 +461,6 @@ impl Connection {
|
||||
Ok(jobs.into_iter().map(Into::into).collect())
|
||||
}
|
||||
|
||||
/// Describe a single server-side job by id. `null` when the server has
|
||||
/// no such job.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn get_job(
|
||||
&self,
|
||||
job_id: String,
|
||||
) -> napi::Result<Option<crate::job::JobDescription>> {
|
||||
let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
|
||||
Ok(description.map(Into::into))
|
||||
}
|
||||
|
||||
/// Request cancellation of a server-side job by id. Returns true if the
|
||||
/// server accepted the cancellation, false if no such job exists.
|
||||
#[napi(catch_unwind)]
|
||||
@@ -477,34 +468,6 @@ impl Connection {
|
||||
self.get_inner()?.cancel_job(&job_id).await.default_error()
|
||||
}
|
||||
|
||||
/// The lifecycle event history of a server-side job (all jobs when
|
||||
/// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
|
||||
/// no history.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
|
||||
let batches = self
|
||||
.get_inner()?
|
||||
.job_history(job_id.as_deref())
|
||||
.await
|
||||
.default_error()?;
|
||||
let Some(first) = batches.first() else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
for batch in &batches {
|
||||
writer
|
||||
.write(batch)
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
}
|
||||
writer
|
||||
.finish()
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
drop(writer);
|
||||
Ok(Buffer::from(out))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
/// Describe a namespace and return its properties.
|
||||
pub async fn describe_namespace(
|
||||
|
||||
+90
-34
@@ -3,6 +3,9 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::RecordBatch;
|
||||
use lancedb::job::JobEventsRequest;
|
||||
use napi::bindgen_prelude::Buffer;
|
||||
use napi_derive::napi;
|
||||
|
||||
use crate::error::NapiErrorExt;
|
||||
@@ -55,12 +58,98 @@ impl Job {
|
||||
pub async fn cancel(&self) -> napi::Result<()> {
|
||||
self.inner.cancel().await.default_error()
|
||||
}
|
||||
|
||||
/// Ask the backend for this job's current state, and for a server-side job
|
||||
/// its full record, then cache it for the getters below.
|
||||
///
|
||||
/// They are all null until this runs, because submitting an operation
|
||||
/// returns only a job id. {@link Job.status} fetches the whole record too;
|
||||
/// {@link Job.wait} records only the terminal state it establishes.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn refresh(&self) -> napi::Result<()> {
|
||||
self.inner.refresh().await.default_error()
|
||||
}
|
||||
|
||||
/// The last observed lifecycle state, without contacting the backend.
|
||||
#[napi(getter)]
|
||||
pub fn state(&self) -> Option<String> {
|
||||
self.inner.state()
|
||||
}
|
||||
|
||||
/// The job's type, as the server names it. Null for an in-process job,
|
||||
/// which has no server-side record.
|
||||
#[napi(getter)]
|
||||
pub fn job_type(&self) -> Option<String> {
|
||||
self.inner.job_type()
|
||||
}
|
||||
|
||||
/// When the job was created, in milliseconds since the epoch.
|
||||
#[napi(getter)]
|
||||
pub fn creation_ms(&self) -> Option<i64> {
|
||||
self.inner.creation_ms()
|
||||
}
|
||||
|
||||
/// The job-type-specific specification as a JSON string, when present.
|
||||
#[napi(getter)]
|
||||
pub fn spec_json(&self) -> Option<String> {
|
||||
self.inner.spec().map(|spec| spec.to_string())
|
||||
}
|
||||
|
||||
/// The job-type-specific terminal result as a JSON string. Null until the
|
||||
/// job succeeds, so a job that never terminates reports its progress
|
||||
/// through {@link Job.events} instead.
|
||||
#[napi(getter)]
|
||||
pub fn result_json(&self) -> Option<String> {
|
||||
self.inner.result().map(|result| result.to_string())
|
||||
}
|
||||
|
||||
/// Why the job failed, when it failed and the server reports a reason.
|
||||
#[napi(getter)]
|
||||
pub fn failure(&self) -> Option<JobFailureInfo> {
|
||||
self.inner.failure().map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
})
|
||||
}
|
||||
|
||||
/// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
|
||||
/// The TypeScript wrapper turns it into an Arrow table.
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
|
||||
let batches = self
|
||||
.inner
|
||||
.events(JobEventsRequest { limit, filter })
|
||||
.await
|
||||
.default_error()?;
|
||||
batches_to_ipc_buffer(&batches)
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
|
||||
pub(crate) fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
|
||||
let Some(first) = batches.first() else {
|
||||
return Ok(Buffer::from(Vec::<u8>::new()));
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
for batch in batches {
|
||||
writer
|
||||
.write(batch)
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
}
|
||||
writer
|
||||
.finish()
|
||||
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
|
||||
drop(writer);
|
||||
Ok(Buffer::from(out))
|
||||
}
|
||||
|
||||
/// A row from `Connection.listJobs`: one server-side job.
|
||||
#[napi(object)]
|
||||
pub struct JobInfo {
|
||||
/// The job id -- what `Connection.getJob` and `Connection.cancelJob`
|
||||
/// The job id -- what `Connection.openJob` and `Connection.cancelJob`
|
||||
/// accept.
|
||||
pub job_id: String,
|
||||
/// The table the job runs against, without URI or namespace.
|
||||
@@ -91,36 +180,3 @@ pub struct JobFailureInfo {
|
||||
pub message: Option<String>,
|
||||
pub retryable: Option<bool>,
|
||||
}
|
||||
|
||||
/// A described job from `Connection.getJob`.
|
||||
#[napi(object)]
|
||||
pub struct JobDescription {
|
||||
pub job_id: String,
|
||||
pub job_type: String,
|
||||
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
|
||||
pub state: String,
|
||||
/// When the job was created, in milliseconds since the epoch.
|
||||
pub creation_ms: i64,
|
||||
/// The job-type-specific specification as a JSON string, when present.
|
||||
pub spec_json: Option<String>,
|
||||
/// Why the job failed, when the job is failed and the server reports a
|
||||
/// reason.
|
||||
pub failure: Option<JobFailureInfo>,
|
||||
}
|
||||
|
||||
impl From<lancedb::database::JobDescription> for JobDescription {
|
||||
fn from(description: lancedb::database::JobDescription) -> Self {
|
||||
Self {
|
||||
job_id: description.job_id,
|
||||
job_type: description.job_type,
|
||||
state: description.state,
|
||||
creation_ms: description.creation_ms,
|
||||
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
|
||||
failure: description.failure.map(|failure| JobFailureInfo {
|
||||
phase: failure.phase,
|
||||
message: failure.message,
|
||||
retryable: failure.retryable,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user