Compare commits

..
Author SHA1 Message Date
dependabot[bot] cacf972b97 build(deps): bump prost in the rust-minor-patch group
Bumps the rust-minor-patch group with 1 update: [prost](https://github.com/tokio-rs/prost).


Updates `prost` from 0.14.3 to 0.14.4
- [Release notes](https://github.com/tokio-rs/prost/releases)
- [Changelog](https://github.com/tokio-rs/prost/blob/master/CHANGELOG.md)
- [Commits](https://github.com/tokio-rs/prost/compare/v0.14.3...v0.14.4)

---
updated-dependencies:
- dependency-name: prost
  dependency-version: 0.14.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: rust-minor-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
2026-09-05 15:00:16 +00:00
61 changed files with 817 additions and 3808 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.39.0-beta.6"
current_version = "0.39.0-beta.1"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Generated
+51 -53
View File
@@ -3526,8 +3526,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4886,8 +4886,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arc-swap",
"arrow",
@@ -4959,8 +4959,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4982,7 +4982,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4996,7 +4996,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5005,8 +5005,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrayref",
"crunchy",
@@ -5016,8 +5016,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5054,8 +5054,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow",
"arrow-array",
@@ -5085,8 +5085,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow",
"arrow-array",
@@ -5103,8 +5103,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"proc-macro2",
"quote",
@@ -5113,8 +5113,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5147,8 +5147,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5179,8 +5179,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arc-swap",
"arrow",
@@ -5244,8 +5244,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5267,8 +5267,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow",
"arrow-array",
@@ -5308,8 +5308,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5323,23 +5323,21 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow",
"async-trait",
"bytes",
"lance-core",
"lance-namespace-reqwest-client",
"serde",
"serde_json",
"snafu 0.9.0",
]
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5378,9 +5376,9 @@ dependencies = [
[[package]]
name = "lance-namespace-reqwest-client"
version = "0.12.0"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af"
checksum = "1d06b1fbb5d41f93bc652b61e2872af92e8a6c5f6b4ce8839a8ecfa05365d359"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -5392,8 +5390,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5407,8 +5405,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow",
"arrow-array",
@@ -5448,8 +5446,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5462,8 +5460,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
version = "12.0.0-beta.11"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.11#4a0e26895729feb86d0cb9c09d551bfd619c6472"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5476,7 +5474,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.39.0-beta.5"
version = "0.39.0-beta.1"
dependencies = [
"ahash",
"anyhow",
@@ -5567,7 +5565,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.39.0-beta.5"
version = "0.39.0-beta.1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5592,7 +5590,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.39.0-beta.5"
version = "0.39.0-beta.1"
dependencies = [
"arrow",
"async-trait",
@@ -7703,9 +7701,9 @@ dependencies = [
[[package]]
name = "prost"
version = "0.14.3"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
dependencies = [
"bytes",
"prost-derive",
@@ -7732,9 +7730,9 @@ dependencies = [
[[package]]
name = "prost-derive"
version = "0.14.3"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.14.0",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.11", default-features = false, "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.11", "tag" = "v12.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.39.0-beta.6</version>
<version>0.39.0-beta.1</version>
</dependency>
```
+62 -24
View File
@@ -448,6 +448,26 @@ on the returned job to know when cleanup has finished.
***
### getJob()
```ts
abstract getJob(jobId): Promise<null | JobDescription>
```
Describe a single server-side job by id.
Resolves to `null` when the server has no such job.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;`null` \| [`JobDescription`](../interfaces/JobDescription.md)&gt;
***
### isOpen()
```ts
@@ -462,6 +482,48 @@ Return true if the connection has not been closed
***
### job()
```ts
abstract job(jobId): Job
```
A [Job](Job.md) handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect on
the job itself.
#### Parameters
* **jobId**: `string`
#### Returns
[`Job`](Job.md)
***
### jobHistory()
```ts
abstract jobHistory(jobId?): Promise<Table<any>>
```
The lifecycle event history of a server-side job, as an Arrow table.
Lists history across all jobs when `jobId` is omitted.
#### Parameters
* **jobId?**: `string`
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### listJobs()
```ts
@@ -586,30 +648,6 @@ A page of table names and an
***
### openJob()
```ts
abstract openJob(jobId): Promise<Job>
```
Open a server-side job by id, returning a handle with its record already
populated. Rejects when the server has no such job, the way
[Connection.openTable](Connection.md#opentable) does for a missing table.
The returned [Job](Job.md) answers for its own state, specification,
result, failure and event history, so there is no separate
connection-level call for any of them.
#### Parameters
* **jobId**: `string`
#### Returns
`Promise`&lt;[`Job`](Job.md)&gt;
***
### openMaterializedView()
```ts
+16 -163
View File
@@ -8,116 +8,28 @@
A handle to an operation that may still be running.
The operation may already be complete when the handle is created.
## Constructors
The detail getters read what the handle last observed. Submitting an
operation returns only a job id, so populating them eagerly would cost an
extra round trip on every call:
### new Job()
- [Job.refresh](Job.md#refresh) and [Job.status](Job.md#status) fetch the whole record.
- [Job.wait](Job.md#wait) records the terminal state it establishes, but not the
rest of the record.
- Everything is null until one of those runs.
```ts
new Job(): Job
```
#### Returns
[`Job`](Job.md)
## Accessors
### creationMs
```ts
get creationMs(): null | number
```
When the job was created, in milliseconds since the epoch.
#### Returns
`null` \| `number`
***
### failure
```ts
get failure(): null | JobFailureInfo
```
Why the job failed, when it failed and the server reports a reason.
#### Returns
`null` \| [`JobFailureInfo`](../interfaces/JobFailureInfo.md)
***
### id
```ts
get id(): null | string
```
Identifies the operation on the server that is running it.
Operations that run in this process have no server id. The value is
opaque: parsing it or storing it to resume the job later is not supported.
#### Returns
`null` \| `string`
***
### jobType
```ts
get jobType(): null | string
```
The job's type, as the server names it. Null for an in-process job, which
has no server-side record.
#### Returns
`null` \| `string`
***
### result
```ts
get result(): any
```
The job-type-specific terminal result. Null until the job succeeds, so a
job that never terminates reports its progress through [Job.events](Job.md#events)
instead.
#### Returns
`any`
***
### spec
```ts
get spec(): any
```
The job-type-specific specification it was submitted with.
#### Returns
`any`
***
### state
```ts
get state(): null | string
```
The last observed lifecycle state, without contacting the backend.
Identifies the operation on the server that is running it. Operations
that run in this process have no server id. The value is opaque.
#### Returns
@@ -139,61 +51,18 @@ Request cancellation. Cancelling a finished operation is a no-op.
***
### events()
```ts
events(options?): Promise<Table<any>>
```
This job's recorded lifecycle events.
Where the getters above report a terminal result only once the job reaches
one, events are written as the job runs and outlive the workers that
produced them. A distributed job records a `claim`/`claim_complete` pair
per unit of work, each carrying `rows_processed`, so a job that never
finishes still accounts for what it did.
The server caps results at 1000 rows by default and 10,000 at most, and
truncates without saying so, so pass `limit` for a job that emits an event
per fragment. `filter` is a SQL-like expression over the `state`,
`updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
#### Parameters
* **options?**: [`JobEventsOptions`](../interfaces/JobEventsOptions.md)
#### Returns
`Promise`&lt;`Table`&lt;`any`&gt;&gt;
***
### refresh()
```ts
refresh(): Promise<void>
```
Ask the backend for this job's current state, and for a server-side job
its full record, then cache it for the getters above.
#### Returns
`Promise`&lt;`void`&gt;
***
### status()
```ts
status(): Promise<string>
```
The operation's current lifecycle state: "running", "finished", "failed",
or "cancelled".
The operation's current lifecycle state: "running", "finished",
"failed", or "cancelled".
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject on a
terminal failure state. Also refreshes the getters above.
A point snapshot; unlike [Job.wait](Job.md#wait) it does not block or reject
on a terminal failure state. States a newer server reports that this
client version does not know pass through as-is.
#### Returns
@@ -201,22 +70,6 @@ terminal failure state. Also refreshes the getters above.
***
### toString()
```ts
toString(): string
```
Every field the handle currently knows, one per line, with the JSON
payloads indented -- a refresh job's spec and result are the point of
printing it.
#### Returns
`string`
***
### wait()
```ts
+1 -1
View File
@@ -96,7 +96,7 @@
- [IvfFlatOptions](interfaces/IvfFlatOptions.md)
- [IvfPqOptions](interfaces/IvfPqOptions.md)
- [IvfRqOptions](interfaces/IvfRqOptions.md)
- [JobEventsOptions](interfaces/JobEventsOptions.md)
- [JobDescription](interfaces/JobDescription.md)
- [JobFailureInfo](interfaces/JobFailureInfo.md)
- [JobInfo](interfaces/JobInfo.md)
- [ListNamespacesOptions](interfaces/ListNamespacesOptions.md)
+66
View File
@@ -0,0 +1,66 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobDescription
# Interface: JobDescription
A described job from `Connection.getJob`.
## Properties
### creationMs
```ts
creationMs: number;
```
When the job was created, in milliseconds since the epoch.
***
### failure?
```ts
optional failure: JobFailureInfo;
```
Why the job failed, when the job is failed and the server reports a
reason.
***
### jobId
```ts
jobId: string;
```
***
### jobType
```ts
jobType: string;
```
***
### specJson?
```ts
optional specJson: string;
```
The job-type-specific specification as a JSON string, when present.
***
### state
```ts
state: string;
```
Lifecycle state: "running", "finished", "failed", or "cancelled".
@@ -1,29 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / JobEventsOptions
# Interface: JobEventsOptions
Which of a job's events [Job.events](../classes/Job.md#events) returns.
## Properties
### filter?
```ts
optional filter: string;
```
SQL-like filter over the event columns.
***
### limit?
```ts
optional limit: number;
```
Maximum event rows to return, up to the server maximum of 10,000.
+1 -1
View File
@@ -26,7 +26,7 @@ When the job was created, in milliseconds since the epoch.
jobId: string;
```
The job id -- what `Connection.openJob` and `Connection.cancelJob`
The job id -- what `Connection.getJob` and `Connection.cancelJob`
accept.
***
-12
View File
@@ -157,12 +157,6 @@ listing a storage directory.
::: lancedb.job.AsyncJob
::: lancedb.job.JobInfo
::: lancedb.job.JobDescription
::: lancedb.job.JobFailureInfo
::: lancedb.sql.Query
::: lancedb.sql.AsyncQuery
@@ -316,12 +310,6 @@ still work. Queries return descriptors. Call
::: lancedb.exceptions.MissingColumnError
::: lancedb.exceptions.JobNotFoundError
::: lancedb.exceptions.JobFailedError
::: lancedb.exceptions.JobCancelledError
## Integrations
## Pydantic
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.39.0-beta.6</version>
<version>0.39.0-beta.1</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.39.0-beta.6</version>
<version>0.39.0-beta.1</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.15</lance-core.version>
<lance-core.version>12.0.0-beta.11</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.39.0-beta.6"
version = "0.39.0-beta.1"
publish = false
license.workspace = true
description.workspace = true
+11 -66
View File
@@ -939,7 +939,6 @@ describe("remote connection jobs surface", () => {
const { tableFromArrays, tableToIPC } = await import("apache-arrow");
const eventsTable = tableFromArrays({ state: ["created", "succeeded"] });
const eventsBody = Buffer.from(tableToIPC(eventsTable, "stream"));
const queryEventsPayloads: Record<string, unknown>[] = [];
await withMockDatabase(
(req, res) => {
@@ -968,16 +967,6 @@ describe("remote connection jobs surface", () => {
);
}
} else if (req.url === "/v1/jobs/describe") {
if (payload["job_id"] === "job-2") {
res
.writeHead(200, { "Content-Type": "application/json" })
.end(
'{"job_id": "job-2", "job_type": "refresh_column", ' +
'"job_state": "DONE", "creation_ms": 2000, ' +
'"result": {"rows_assigned": 1000000}}',
);
return;
}
if (payload["job_id"] !== "job-1") {
res.writeHead(404).end("no such job");
return;
@@ -999,7 +988,6 @@ describe("remote connection jobs surface", () => {
.writeHead(200, { "Content-Type": "application/json" })
.end('{"job_id": "job-1"}');
} else if (req.url === "/v1/jobs/query_events") {
queryEventsPayloads.push(payload);
res
.writeHead(200, {
"Content-Type": "application/vnd.apache.arrow.stream",
@@ -1016,65 +1004,22 @@ describe("remote connection jobs surface", () => {
expect(jobs[0].state).toEqual("running");
expect(jobs[1].state).toEqual("finished");
const description = await db.getJob("job-1");
expect(description?.state).toEqual("failed");
expect(JSON.parse(description?.specJson ?? "")).toEqual({
column: "vec",
});
expect(description?.failure?.message).toEqual("worker died");
expect(await db.getJob("missing")).toBeNull();
expect(await db.cancelJob("job-1")).toBe(true);
expect(await db.cancelJob("missing")).toBe(false);
// Opening a job hands back a populated handle; a missing one rejects.
await expect(db.openJob("missing")).rejects.toThrow("not found");
const finished = await db.openJob("job-2");
expect(finished.state).toEqual("finished");
expect(finished.result).toEqual({
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
rows_assigned: 1000000,
});
const history = await db.jobHistory("job-1");
expect(history.numRows).toEqual(2);
const job = await db.openJob("job-1");
const job = db.job("job-1");
expect(job.id).toEqual("job-1");
// openJob already populated the handle; refresh() re-reads it.
expect(job.state).toEqual("failed");
await job.refresh();
expect(job.state).toEqual("failed");
expect(job.jobType).toEqual("create_index");
expect(job.creationMs).toEqual(1000);
expect(job.spec).toEqual({ column: "vec" });
expect(job.result).toBeNull();
expect(job.failure?.message).toEqual("worker died");
// The handle reaches its own events, supplying its job id.
const jobEvents = await job.events({
limit: 500,
filter: "state = 'claim_complete'",
});
expect(jobEvents.numRows).toEqual(2);
expect(queryEventsPayloads.pop()).toEqual({
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
job_id: "job-1",
limit: 500,
filter: "state = 'claim_complete'",
});
// Printing lays every known field out on its own line, with the JSON
// payloads indented rather than crammed onto one line.
expect(`${job}`).toEqual(
[
"Job(",
' id="job-1",',
' state="failed",',
' jobType="create_index",',
" creationMs=1000,",
" spec={",
' "column": "vec"',
" },",
" failure={",
' "phase": "execute",',
' "message": "worker died",',
' "retryable": true',
" },",
")",
].join("\n"),
);
expect(await job.status()).toEqual("failed");
await expect(job.wait()).rejects.toThrow("worker died");
},
+2 -2
View File
@@ -281,7 +281,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
numIndices: 0,
numRows: 3,
// Full on-disk size of the two data files, footers and metadata included.
totalBytes: 550,
totalBytes: 684,
});
// Index files count toward totalBytes too (only deletion files and
@@ -289,7 +289,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await table.createIndex("id", { config: Index.btree() });
const statsWithIndex = await table.stats();
expect(statsWithIndex.numIndices).toBe(1);
expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
});
it("should overwrite data if asked", async () => {
+37 -11
View File
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { tableFromIPC } from "apache-arrow";
import {
Data,
SchemaLike,
@@ -15,7 +16,6 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { Job } from "./job";
import {
MaterializedView,
MaterializedViewSelect,
@@ -27,6 +27,8 @@ import type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
DropNamespaceResponse,
Job,
JobDescription,
JobInfo,
ListNamespacesResponse,
ListTablesResponse,
@@ -555,19 +557,24 @@ export abstract class Connection {
): Promise<void>;
/**
* Open a server-side job by id, returning a handle with its record already
* populated. Rejects when the server has no such job, the way
* {@link Connection.openTable} does for a missing table.
* A {@link Job} handle for a server-side job by id.
*
* The returned {@link Job} answers for its own state, specification,
* result, failure and event history, so there is no separate
* connection-level call for any of them.
* The handle is constructed without a server round trip; an unknown id
* surfaces when the handle is used. Dropping the handle has no effect on
* the job itself.
*/
abstract openJob(jobId: string): Promise<Job>;
abstract job(jobId: string): Job;
/** List server-side jobs across the database's tables. */
abstract listJobs(): Promise<JobInfo[]>;
/**
* Describe a single server-side job by id.
*
* Resolves to `null` when the server has no such job.
*/
abstract getJob(jobId: string): Promise<JobDescription | null>;
/**
* Request cancellation of a server-side job by id.
*
@@ -575,6 +582,13 @@ export abstract class Connection {
* such job exists. Cancelling an already-terminal job is a no-op success.
*/
abstract cancelJob(jobId: string): Promise<boolean>;
/**
* The lifecycle event history of a server-side job, as an Arrow table.
*
* Lists history across all jobs when `jobId` is omitted.
*/
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
}
/** @hideconstructor */
@@ -855,7 +869,7 @@ export class LocalConnection extends Connection {
}
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
return new Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
return this.inner.dropTableAsync(name, namespacePath ?? []);
}
async dropAllTables(namespacePath?: string[]): Promise<void> {
@@ -914,17 +928,29 @@ export class LocalConnection extends Connection {
);
}
async openJob(jobId: string): Promise<Job> {
return new Job(await this.inner.openJob(jobId));
job(jobId: string): Job {
return this.inner.job(jobId);
}
async listJobs(): Promise<JobInfo[]> {
return this.inner.listJobs();
}
async getJob(jobId: string): Promise<JobDescription | null> {
return this.inner.getJob(jobId);
}
async cancelJob(jobId: string): Promise<boolean> {
return this.inner.cancelJob(jobId);
}
async jobHistory(jobId?: string): Promise<ArrowTable> {
const buf = await this.inner.jobHistory(jobId);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
}
/**
+7 -3
View File
@@ -94,9 +94,13 @@ export {
RenameTableOptions,
} from "./connection";
export { JobFailureInfo, JobInfo, Session } from "./native.js";
export { Job, JobEventsOptions } from "./job";
export {
Job,
JobDescription,
JobFailureInfo,
JobInfo,
Session,
} from "./native.js";
export {
AutoQuery,
-188
View File
@@ -1,188 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Table as ArrowTable, tableFromIPC } from "apache-arrow";
import { JobFailureInfo, Job as NativeJob } from "./native";
/** Which of a job's events {@link Job.events} returns. */
export interface JobEventsOptions {
/** Maximum event rows to return, up to the server maximum of 10,000. */
limit?: number;
/** SQL-like filter over the event columns. */
filter?: string;
}
/**
* A handle to an operation that may still be running.
*
* The operation may already be complete when the handle is created.
*
* The detail getters read what the handle last observed. Submitting an
* operation returns only a job id, so populating them eagerly would cost an
* extra round trip on every call:
*
* - {@link Job.refresh} and {@link Job.status} fetch the whole record.
* - {@link Job.wait} records the terminal state it establishes, but not the
* rest of the record.
* - Everything is null until one of those runs.
*
* @hideconstructor
*/
export class Job {
private readonly inner: NativeJob;
constructor(inner: NativeJob) {
this.inner = inner;
}
/**
* Identifies the operation on the server that is running it.
*
* Operations that run in this process have no server id. The value is
* opaque: parsing it or storing it to resume the job later is not supported.
*/
get id(): string | null {
return this.inner.id ?? null;
}
/** The last observed lifecycle state, without contacting the backend. */
get state(): string | null {
return this.inner.state ?? null;
}
/**
* The job's type, as the server names it. Null for an in-process job, which
* has no server-side record.
*/
get jobType(): string | null {
return this.inner.jobType ?? null;
}
/** When the job was created, in milliseconds since the epoch. */
get creationMs(): number | null {
return this.inner.creationMs ?? null;
}
/** The job-type-specific specification it was submitted with. */
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get spec(): any | null {
return parseJson(this.inner.specJson);
}
/**
* The job-type-specific terminal result. Null until the job succeeds, so a
* job that never terminates reports its progress through {@link Job.events}
* instead.
*/
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
get result(): any | null {
return parseJson(this.inner.resultJson);
}
/** Why the job failed, when it failed and the server reports a reason. */
get failure(): JobFailureInfo | null {
return this.inner.failure ?? null;
}
/**
* The operation's current lifecycle state: "running", "finished", "failed",
* or "cancelled".
*
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
* terminal failure state. Also refreshes the getters above.
*/
async status(): Promise<string> {
return this.inner.status();
}
/** Wait until the operation reaches a terminal state. */
async wait(): Promise<void> {
return this.inner.wait();
}
/** Request cancellation. Cancelling a finished operation is a no-op. */
async cancel(): Promise<void> {
return this.inner.cancel();
}
/**
* Ask the backend for this job's current state, and for a server-side job
* its full record, then cache it for the getters above.
*/
async refresh(): Promise<void> {
return this.inner.refresh();
}
/**
* This job's recorded lifecycle events.
*
* Where the getters above report a terminal result only once the job reaches
* one, events are written as the job runs and outlive the workers that
* produced them. A distributed job records a `claim`/`claim_complete` pair
* per unit of work, each carrying `rows_processed`, so a job that never
* finishes still accounts for what it did.
*
* The server caps results at 1000 rows by default and 10,000 at most, and
* truncates without saying so, so pass `limit` for a job that emits an event
* per fragment. `filter` is a SQL-like expression over the `state`,
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
*/
async events(options?: JobEventsOptions): Promise<ArrowTable> {
const buf = await this.inner.events(options?.limit, options?.filter);
if (buf.length === 0) {
return new ArrowTable();
}
return tableFromIPC(buf);
}
/**
* Every field the handle currently knows, one per line, with the JSON
* payloads indented -- a refresh job's spec and result are the point of
* printing it.
*/
toString(): string {
if (this.state === null) {
const known = this.id === null ? "" : `id=${JSON.stringify(this.id)}, `;
return `Job(${known}not refreshed)`;
}
const fields: string[] = [];
if (this.id !== null) {
fields.push(`id=${JSON.stringify(this.id)}`);
}
fields.push(`state=${JSON.stringify(this.state)}`);
if (this.jobType !== null) {
fields.push(`jobType=${JSON.stringify(this.jobType)}`);
}
if (this.creationMs !== null) {
fields.push(`creationMs=${this.creationMs}`);
}
for (const [name, value] of [
["spec", this.spec],
["result", this.result],
] as const) {
if (value !== null) {
fields.push(`${name}=${indentJson(value)}`);
}
}
if (this.failure !== null) {
fields.push(`failure=${indentJson(this.failure)}`);
}
return `Job(${fields.map((field) => `\n${REPR_INDENT}${field},`).join("")}\n)`;
}
[Symbol.for("nodejs.util.inspect.custom")](): string {
return this.toString();
}
}
const REPR_INDENT = " ";
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function indentJson(value: any): string {
return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
}
// biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
function parseJson(raw: string | null | undefined): any | null {
return raw === null || raw === undefined ? null : JSON.parse(raw);
}
+9 -11
View File
@@ -19,7 +19,6 @@ import {
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import { IndexOptions } from "./indices";
import { Job } from "./job";
import { MergeInsertBuilder } from "./merge";
import {
AddColumnsResult,
@@ -31,6 +30,7 @@ import {
DropColumnsResult,
IndexConfig,
IndexStatistics,
Job,
LsmStats,
Branches as NativeBranches,
OptimizeStats,
@@ -1124,15 +1124,13 @@ export class LocalTable extends Table {
): Promise<Job> {
// biome-ignore lint/suspicious/noExplicitAny: skip
const nativeIndex = (options?.config as any)?.inner;
return new Job(
await this.inner.createIndexAsync(
nativeIndex,
column,
options?.replace,
options?.waitTimeoutSeconds,
options?.name,
options?.train,
),
return await this.inner.createIndexAsync(
nativeIndex,
column,
options?.replace,
options?.waitTimeoutSeconds,
options?.name,
options?.train,
);
}
@@ -1315,7 +1313,7 @@ export class LocalTable extends Table {
}
async refreshColumnAsync(column: string): Promise<Job> {
return new Job(await this.inner.refreshColumnAsync(column));
return await this.inner.refreshColumnAsync(column);
}
async refreshMaterializedView(
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.39.0-beta.6",
"version": "0.39.0-beta.1",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
+24 -24
View File
@@ -41,7 +41,7 @@ importers:
version: 3.7.0(@emnapi/core@1.10.0)(@emnapi/runtime@1.11.3)(@types/node@22.7.4)
'@opentelemetry/sdk-metrics':
specifier: ^2.10.0
version: 2.11.0(@opentelemetry/api@1.9.1)
version: 2.10.0(@opentelemetry/api@1.9.1)
'@types/axios':
specifier: ^0.14.0
version: 0.14.4
@@ -80,7 +80,7 @@ importers:
version: 0.2.7
ts-jest:
specifier: ^29.1.2
version: 29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4)
version: 29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4)
typedoc:
specifier: 0.26.4
version: 0.26.4(typescript@5.5.4)
@@ -1394,20 +1394,20 @@ packages:
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
engines: {node: '>=8.0.0'}
'@opentelemetry/core@2.11.0':
resolution: {integrity: sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==}
'@opentelemetry/core@2.10.0':
resolution: {integrity: sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.0.0 <1.10.0'
'@opentelemetry/resources@2.11.0':
resolution: {integrity: sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==}
'@opentelemetry/resources@2.10.0':
resolution: {integrity: sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.3.0 <1.10.0'
'@opentelemetry/sdk-metrics@2.11.0':
resolution: {integrity: sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==}
'@opentelemetry/sdk-metrics@2.10.0':
resolution: {integrity: sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==}
engines: {node: ^18.19.0 || >=20.6.0}
peerDependencies:
'@opentelemetry/api': '>=1.9.0 <1.10.0'
@@ -1480,7 +1480,6 @@ packages:
'@smithy/core@3.24.1':
resolution: {integrity: sha512-3mT7o4qQyUWttYnVK3A0Z/u3Xha3E81tXn32Tz6vjZiUXhBrkEivpw1hBYfh84iFF9CSzkBU9Y1DJ3Q6RQ231g==}
engines: {node: '>=18.0.0'}
deprecated: Deprecated due to bug in browser bundling instructions https://github.com/smithy-lang/smithy-typescript/issues/2025
'@smithy/credential-provider-imds@4.3.1':
resolution: {integrity: sha512-0S/acwHnqX4WrjXzhdiDRxsG2s9SC0cpPIK9nZ1R6UOHd+j7uL28+4bHu22urbLk2TVw3fkp6na/+fkUt/pLNQ==}
@@ -3239,8 +3238,8 @@ packages:
peerDependencies:
typescript: '>=4.2.0'
ts-jest@29.4.12:
resolution: {integrity: sha512-Ov6ClY53Fflh6BGAnY2DlTq1hYDrTycz2PVTXBWFW2CU+9zrEqAp9fWdGXl42EXO5RLSFAcAZ2JFKbP+zBTFfw==}
ts-jest@29.4.9:
resolution: {integrity: sha512-LTb9496gYPMCqjeDLdPrKuXtncudeV1yRZnF4Wo5l3SFi0RYEnYRNgMrFIdg+FHvfzjCyQk1cLncWVqiSX+EvQ==}
engines: {node: ^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0}
hasBin: true
peerDependencies:
@@ -5111,22 +5110,22 @@ snapshots:
'@opentelemetry/api@1.9.1': {}
'@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)':
'@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1)':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/semantic-conventions': 1.43.0
'@opentelemetry/resources@2.11.0(@opentelemetry/api@1.9.1)':
'@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1)':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1)
'@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1)
'@opentelemetry/semantic-conventions': 1.43.0
'@opentelemetry/sdk-metrics@2.11.0(@opentelemetry/api@1.9.1)':
'@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1)':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1)
'@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1)
'@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1)
'@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1)
'@opentelemetry/semantic-conventions@1.43.0': {}
@@ -5575,7 +5574,7 @@ snapshots:
globby: 11.1.0
is-glob: 4.0.3
minimatch: 9.0.9
semver: 7.8.5
semver: 7.8.0
ts-api-utils: 1.4.3(typescript@5.5.4)
optionalDependencies:
typescript: 5.5.4
@@ -6427,7 +6426,7 @@ snapshots:
'@babel/parser': 7.29.3
'@istanbuljs/schema': 0.1.6
istanbul-lib-coverage: 3.2.2
semver: 7.8.5
semver: 7.8.0
transitivePeerDependencies:
- supports-color
@@ -6706,7 +6705,7 @@ snapshots:
jest-util: 29.7.0
natural-compare: 1.4.0
pretty-format: 29.7.0
semver: 7.8.5
semver: 7.8.0
transitivePeerDependencies:
- supports-color
@@ -6829,7 +6828,7 @@ snapshots:
make-dir@4.0.0:
dependencies:
semver: 7.8.5
semver: 7.8.0
make-error@1.3.6: {}
@@ -7163,7 +7162,8 @@ snapshots:
semver@7.8.0: {}
semver@7.8.5: {}
semver@7.8.5:
optional: true
sharp@0.35.4(@types/node@22.7.4):
dependencies:
@@ -7327,7 +7327,7 @@ snapshots:
dependencies:
typescript: 5.5.4
ts-jest@29.4.12(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4):
ts-jest@29.4.9(@babel/core@7.29.7)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.29.7))(jest-util@29.7.0)(jest@29.7.0(@types/node@22.7.4))(typescript@5.5.4):
dependencies:
bs-logger: 0.2.6
fast-json-stable-stringify: 2.1.0
@@ -7336,7 +7336,7 @@ snapshots:
json5: 2.2.3
lodash.memoize: 4.1.2
make-error: 1.3.6
semver: 7.8.5
semver: 7.8.0
type-fest: 4.41.0
typescript: 5.5.4
yargs-parser: 21.1.1
+45 -8
View File
@@ -442,15 +442,13 @@ impl Connection {
self.get_inner()?.drop_all_tables(&ns).await.default_error()
}
/// Open a server-side job by id, returning a handle with its record
/// already populated. Rejects when the server has no such job.
/// A `Job` handle for a server-side job by id.
///
/// The returned handle answers for its own state, specification, result,
/// failure and event history, so there is no separate connection-level
/// call for any of them.
#[napi(catch_unwind)]
pub async fn open_job(&self, job_id: String) -> napi::Result<crate::job::Job> {
let job = self.get_inner()?.open_job(&job_id).await.default_error()?;
/// The handle is constructed without a server round trip; an unknown id
/// surfaces when the handle is used.
#[napi]
pub fn job(&self, job_id: String) -> napi::Result<crate::job::Job> {
let job = self.get_inner()?.job(job_id).default_error()?;
Ok(crate::job::Job::new(job))
}
@@ -461,6 +459,17 @@ impl Connection {
Ok(jobs.into_iter().map(Into::into).collect())
}
/// Describe a single server-side job by id. `null` when the server has
/// no such job.
#[napi(catch_unwind)]
pub async fn get_job(
&self,
job_id: String,
) -> napi::Result<Option<crate::job::JobDescription>> {
let description = self.get_inner()?.get_job(&job_id).await.default_error()?;
Ok(description.map(Into::into))
}
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
#[napi(catch_unwind)]
@@ -468,6 +477,34 @@ impl Connection {
self.get_inner()?.cancel_job(&job_id).await.default_error()
}
/// The lifecycle event history of a server-side job (all jobs when
/// `job_id` is null), as an Arrow IPC stream buffer. Empty when there is
/// no history.
#[napi(catch_unwind)]
pub async fn job_history(&self, job_id: Option<String>) -> napi::Result<Buffer> {
let batches = self
.get_inner()?
.job_history(job_id.as_deref())
.await
.default_error()?;
let Some(first) = batches.first() else {
return Ok(Buffer::from(Vec::<u8>::new()));
};
let mut out = Vec::new();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
for batch in &batches {
writer
.write(batch)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
}
writer
.finish()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
drop(writer);
Ok(Buffer::from(out))
}
#[napi(catch_unwind)]
/// Describe a namespace and return its properties.
pub async fn describe_namespace(
+34 -90
View File
@@ -3,9 +3,6 @@
use std::sync::Arc;
use arrow_array::RecordBatch;
use lancedb::job::JobEventsRequest;
use napi::bindgen_prelude::Buffer;
use napi_derive::napi;
use crate::error::NapiErrorExt;
@@ -58,98 +55,12 @@ impl Job {
pub async fn cancel(&self) -> napi::Result<()> {
self.inner.cancel().await.default_error()
}
/// Ask the backend for this job's current state, and for a server-side job
/// its full record, then cache it for the getters below.
///
/// They are all null until this runs, because submitting an operation
/// returns only a job id. {@link Job.status} fetches the whole record too;
/// {@link Job.wait} records only the terminal state it establishes.
#[napi(catch_unwind)]
pub async fn refresh(&self) -> napi::Result<()> {
self.inner.refresh().await.default_error()
}
/// The last observed lifecycle state, without contacting the backend.
#[napi(getter)]
pub fn state(&self) -> Option<String> {
self.inner.state()
}
/// The job's type, as the server names it. Null for an in-process job,
/// which has no server-side record.
#[napi(getter)]
pub fn job_type(&self) -> Option<String> {
self.inner.job_type()
}
/// When the job was created, in milliseconds since the epoch.
#[napi(getter)]
pub fn creation_ms(&self) -> Option<i64> {
self.inner.creation_ms()
}
/// The job-type-specific specification as a JSON string, when present.
#[napi(getter)]
pub fn spec_json(&self) -> Option<String> {
self.inner.spec().map(|spec| spec.to_string())
}
/// The job-type-specific terminal result as a JSON string. Null until the
/// job succeeds, so a job that never terminates reports its progress
/// through {@link Job.events} instead.
#[napi(getter)]
pub fn result_json(&self) -> Option<String> {
self.inner.result().map(|result| result.to_string())
}
/// Why the job failed, when it failed and the server reports a reason.
#[napi(getter)]
pub fn failure(&self) -> Option<JobFailureInfo> {
self.inner.failure().map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
retryable: failure.retryable,
})
}
/// This job's recorded lifecycle events, as an Arrow IPC stream buffer.
/// The TypeScript wrapper turns it into an Arrow table.
#[napi(catch_unwind)]
pub async fn events(&self, limit: Option<u32>, filter: Option<String>) -> napi::Result<Buffer> {
let batches = self
.inner
.events(JobEventsRequest { limit, filter })
.await
.default_error()?;
batches_to_ipc_buffer(&batches)
}
}
/// Serialise Arrow batches as a single IPC stream for the TypeScript layer.
fn batches_to_ipc_buffer(batches: &[RecordBatch]) -> napi::Result<Buffer> {
let Some(first) = batches.first() else {
return Ok(Buffer::from(Vec::<u8>::new()));
};
let mut out = Vec::new();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut out, &first.schema())
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
for batch in batches {
writer
.write(batch)
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
}
writer
.finish()
.map_err(|e| napi::Error::from_reason(e.to_string()))?;
drop(writer);
Ok(Buffer::from(out))
}
/// A row from `Connection.listJobs`: one server-side job.
#[napi(object)]
pub struct JobInfo {
/// The job id -- what `Connection.openJob` and `Connection.cancelJob`
/// The job id -- what `Connection.getJob` and `Connection.cancelJob`
/// accept.
pub job_id: String,
/// The table the job runs against, without URI or namespace.
@@ -180,3 +91,36 @@ pub struct JobFailureInfo {
pub message: Option<String>,
pub retryable: Option<bool>,
}
/// A described job from `Connection.getJob`.
#[napi(object)]
pub struct JobDescription {
pub job_id: String,
pub job_type: String,
/// Lifecycle state: "running", "finished", "failed", or "cancelled".
pub state: String,
/// When the job was created, in milliseconds since the epoch.
pub creation_ms: i64,
/// The job-type-specific specification as a JSON string, when present.
pub spec_json: Option<String>,
/// Why the job failed, when the job is failed and the server reports a
/// reason.
pub failure: Option<JobFailureInfo>,
}
impl From<lancedb::database::JobDescription> for JobDescription {
fn from(description: lancedb::database::JobDescription) -> Self {
Self {
job_id: description.job_id,
job_type: description.job_type,
state: description.state,
creation_ms: description.creation_ms,
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
failure: description.failure.map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
retryable: failure.retryable,
}),
}
}
}
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.39.0-beta.6"
version = "0.39.0-beta.1"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
+6 -19
View File
@@ -148,13 +148,17 @@ class Connection(object):
start_after: Optional[str],
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
async def open_job(self, job_id: str) -> Job: ...
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def job_history(
self, job_id: Optional[str] = None
) -> List[pa.RecordBatch]: ...
async def execute_query_async(
self,
query: str,
@@ -240,20 +244,9 @@ class BlobFile:
class Job:
@property
def id(self) -> Optional[str]: ...
@property
def _state(self) -> Optional[str]: ...
@property
def _description(self) -> Optional[JobDescription]: ...
async def status(self) -> str: ...
async def wait(self) -> Optional[str]: ...
async def cancel(self) -> None: ...
async def refresh(self) -> None: ...
async def events(
self,
*,
limit: Optional[int] = None,
filter: Optional[str] = None,
) -> pa.Table: ...
class JobInfo:
@property
@@ -285,13 +278,7 @@ class JobDescription:
@property
def creation_ms(self) -> int: ...
@property
def _spec_json(self) -> Optional[str]: ...
@property
def _result_json(self) -> Optional[str]: ...
@property
def spec(self) -> Optional[Any]: ...
@property
def result(self) -> Optional[Any]: ...
def spec_json(self) -> Optional[str]: ...
@property
def failure(self) -> Optional[JobFailureInfo]: ...
+68 -19
View File
@@ -76,7 +76,7 @@ if TYPE_CHECKING:
from .pydantic import LanceModel
from ._lancedb import Connection as LanceDbConnection
from ._lancedb import JobInfo
from ._lancedb import JobDescription, JobInfo
from .common import DATA, URI
from .embeddings import EmbeddingFunctionConfig
from ._lancedb import Session
@@ -745,23 +745,26 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def open_job(self, job_id: str) -> Job:
"""Open a server-side job by id, returning a handle with its record
already populated.
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
The returned [Job][lancedb.job.Job] answers for its own state,
specification, result, failure and event history, so there is no
separate connection-level call for any of them.
Raises `JobNotFoundError` when the server has no such job, the way
`open_table` does for a missing table.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
raise NotImplementedError("open_job is not supported for this connection type")
raise NotImplementedError("job is not supported for this connection type")
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
raise NotImplementedError("list_jobs is not supported for this connection type")
def get_job(self, job_id: str) -> Optional[JobDescription]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
raise NotImplementedError("get_job is not supported for this connection type")
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
@@ -773,6 +776,15 @@ class DBConnection(EnforceOverrides):
"cancel_job is not supported for this connection type"
)
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
raise NotImplementedError(
"job_history is not supported for this connection type"
)
def execute_query(
self,
query: str,
@@ -1450,11 +1462,14 @@ class LanceDBConnection(DBConnection):
)
@override
def open_job(self, job_id: str) -> Job:
"""Open a server-side job by id. See
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
return Job(LOOP.run(self._conn.open_job(job_id)))
return Job(self._conn.job(job_id))
@override
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
@@ -1478,6 +1493,14 @@ class LanceDBConnection(DBConnection):
"""List server-side jobs across the database's tables."""
return LOOP.run(self._conn.list_jobs())
@override
def get_job(self, job_id: str) -> Optional[JobDescription]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
return LOOP.run(self._conn.get_job(job_id))
@override
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
@@ -1488,6 +1511,14 @@ class LanceDBConnection(DBConnection):
"""
return LOOP.run(self._conn.cancel_job(job_id))
@override
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
return LOOP.run(self._conn.job_history(job_id))
@override
def namespace_client(self) -> LanceNamespace:
"""Get the equivalent namespace client for this connection.
@@ -2258,11 +2289,15 @@ class AsyncConnection(object):
namespace_path = []
await self._inner.drop_all_tables(namespace_path=namespace_path)
async def open_job(self, job_id: str) -> AsyncJob:
"""Open a server-side job by id. See
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
def job(self, job_id: str) -> AsyncJob:
"""An [AsyncJob][lancedb.job.AsyncJob] handle for a server-side job
by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
return AsyncJob(await self._inner.open_job(job_id))
return AsyncJob(self._inner.job(job_id))
async def create_function_async(
self, definition: UdfDefinition
@@ -2302,6 +2337,13 @@ class AsyncConnection(object):
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
async def get_job(self, job_id: str) -> Optional[JobDescription]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
return await self._inner.get_job(job_id)
async def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
@@ -2311,6 +2353,13 @@ class AsyncConnection(object):
"""
return await self._inner.cancel_job(job_id)
async def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
return await self._inner.job_history(job_id)
async def execute_query(
self,
query: str,
-6
View File
@@ -35,9 +35,3 @@ class JobCancelledError(RuntimeError):
"""Exception raised when an asynchronous job was cancelled."""
pass
class JobNotFoundError(ValueError):
"""Exception raised when opening a job the server does not have."""
pass
-224
View File
@@ -4,27 +4,15 @@
"""Handles to operations a server may run asynchronously."""
import asyncio
import json
from datetime import timedelta
from typing import Any, Callable, Generic, Optional, TypeVar, cast
import pyarrow as pa
from lancedb.background_loop import LOOP
from . import _lancedb
from ._lancedb import JobDescription, JobFailureInfo, JobInfo
T = TypeVar("T")
__all__ = [
"AsyncJob",
"Job",
"JobDescription",
"JobFailureInfo",
"JobInfo",
]
class AsyncJob(Generic[T]):
"""A handle to an operation that may still be running.
@@ -90,149 +78,6 @@ class AsyncJob(Generic[T]):
return
await self._inner.cancel()
async def refresh(self) -> None:
"""Ask the backend for this job's current state, and for a server-side
job its full record, then cache it for the properties below.
The properties are all `None` until this runs, because submitting an
operation returns only a job id. `status` fetches the whole record too;
`wait` records only the terminal state it establishes.
"""
if self._inner is None:
return
await self._inner.refresh()
@property
def state(self) -> Optional[str]:
"""The last observed lifecycle state, without contacting the backend.
`None` until the handle has talked to it. See :meth:`AsyncJob.refresh`.
"""
if self._inner is None:
return "finished"
return self._inner._state
@property
def job_type(self) -> Optional[str]:
"""The job's type, as the server names it.
`None` for an in-process job, which has no server-side record.
"""
return self._field("job_type")
@property
def creation_ms(self) -> Optional[int]:
"""When the job was created, in milliseconds since the epoch."""
return self._field("creation_ms")
@property
def spec(self) -> Optional[Any]:
"""The job-type-specific specification it was submitted with."""
return self._field("spec")
@property
def result(self) -> Optional[Any]:
"""The job-type-specific terminal result, as reported data rather than
the typed model :meth:`AsyncJob.wait` returns.
`None` until the job succeeds, so a job that never terminates reports
its progress through :meth:`AsyncJob.events` instead.
"""
return self._field("result")
@property
def failure(self) -> Optional[JobFailureInfo]:
"""Why the job failed, when it failed and the server reports a reason."""
return self._field("failure")
@property
def _spec_json(self) -> Optional[str]:
return self._field("_spec_json")
@property
def _result_json(self) -> Optional[str]:
return self._field("_result_json")
def _field(self, name: str) -> Optional[Any]:
description = self._inner._description if self._inner is not None else None
return getattr(description, name) if description is not None else None
async def events(
self,
*,
limit: Optional[int] = None,
filter: Optional[str] = None,
) -> "pa.Table":
"""This job's recorded lifecycle events.
Where the properties above report a terminal result only once the job
reaches one, events are written as the job runs and outlive the workers
that produced them. A distributed job records a `claim`/`claim_complete`
pair per unit of work, each carrying `rows_processed`, so a job that
never finishes still accounts for what it did.
Parameters
----------
limit: int, optional
Maximum event rows to return. The server caps results at 1000 by
default and 10,000 at most, and truncates without saying so, so
pass this for a job that emits an event per fragment.
filter: str, optional
SQL-like expression over the `state`, `updated_by`, `emitted_from`,
`emitted_by`, and `claim_entity` columns, such as
``state = 'claim_complete'``.
"""
if self._inner is None:
raise NotImplementedError(
"job event history is only available for server-side jobs"
)
return await self._inner.events(limit=limit, filter=filter)
def __repr__(self) -> str:
return _job_repr("AsyncJob", self)
_REPR_INDENT = " " * 4
def _repr_payload(value: Any) -> str:
"""Render a job payload as indented JSON, aligned under its field."""
try:
rendered = json.dumps(value, indent=4)
except TypeError:
return repr(value)
return rendered.replace("\n", "\n" + _REPR_INDENT)
def _job_repr(kind: str, job: Any) -> str:
"""Render every field the handle currently knows, omitting the rest.
One field per line, with the JSON payloads indented, because a refresh
job's spec and result are the point of printing it.
"""
state = job.state
if state is None:
# Nothing has been fetched yet, so there is nothing to lay out.
known = f"id={job.id!r}, " if job.id is not None else ""
return f"{kind}({known}not refreshed)"
fields = []
if job.id is not None:
fields.append(f"id={job.id!r}")
fields.append(f"state={state!r}")
for name in ("job_type", "creation_ms"):
value = getattr(job, name)
if value is not None:
fields.append(f"{name}={value!r}")
for name in ("spec", "result"):
value = getattr(job, name)
if value is not None:
fields.append(f"{name}={_repr_payload(value)}")
if job.failure is not None:
fields.append(f"failure={job.failure!r}")
body = "".join(f"\n{_REPR_INDENT}{field}," for field in fields)
return f"{kind}({body}\n)"
class Job(Generic[T]):
"""Synchronous counterpart of `AsyncJob` with the same result type."""
@@ -277,75 +122,6 @@ class Job(Generic[T]):
return
LOOP.run(self._inner.cancel())
def refresh(self) -> None:
"""Ask the backend for this job's current state and record.
See :meth:`AsyncJob.refresh`.
"""
if self._inner is None:
return
LOOP.run(self._inner.refresh())
@property
def state(self) -> Optional[str]:
"""The last observed lifecycle state. See :attr:`AsyncJob.state`."""
return self._inner.state if self._inner is not None else "finished"
@property
def job_type(self) -> Optional[str]:
"""The job's type. See :attr:`AsyncJob.job_type`."""
return self._field("job_type")
@property
def creation_ms(self) -> Optional[int]:
"""When the job was created. See :attr:`AsyncJob.creation_ms`."""
return self._field("creation_ms")
@property
def spec(self) -> Optional[Any]:
"""The job's specification. See :attr:`AsyncJob.spec`."""
return self._field("spec")
@property
def result(self) -> Optional[Any]:
"""The job's terminal result. See :attr:`AsyncJob.result`."""
return self._field("result")
@property
def failure(self) -> Optional[JobFailureInfo]:
"""Why the job failed. See :attr:`AsyncJob.failure`."""
return self._field("failure")
@property
def _spec_json(self) -> Optional[str]:
return self._field("_spec_json")
@property
def _result_json(self) -> Optional[str]:
return self._field("_result_json")
def _field(self, name: str) -> Optional[Any]:
return getattr(self._inner, name) if self._inner is not None else None
def events(
self,
*,
limit: Optional[int] = None,
filter: Optional[str] = None,
) -> "pa.Table":
"""This job's recorded lifecycle events.
See :meth:`AsyncJob.events`.
"""
if self._inner is None:
raise NotImplementedError(
"job event history is only available for server-side jobs"
)
return LOOP.run(self._inner.events(limit=limit, filter=filter))
def __repr__(self) -> str:
return _job_repr("Job", self)
def _typed_job(
inner: "_lancedb.Job", result_decoder: Callable[[str], T]
+24 -5
View File
@@ -31,7 +31,7 @@ from ..sql import QueryDescription
from ..materialized_view import MaterializedView, SelectArg
if TYPE_CHECKING:
from .._lancedb import JobInfo
from .._lancedb import JobDescription, JobInfo
from ..embeddings import EmbeddingFunctionConfig
from lance_namespace import (
LanceNamespace,
@@ -739,11 +739,14 @@ class RemoteDBConnection(DBConnection):
)
@override
def open_job(self, job_id: str) -> Job:
"""Open a server-side job by id. See
[DBConnection.open_job][lancedb.db.DBConnection.open_job].
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
The handle is constructed without a server round trip; an unknown id
surfaces when the handle is used. Dropping the handle has no effect
on the job itself.
"""
return Job(LOOP.run(self._conn.open_job(job_id)))
return Job(self._conn.job(job_id))
@override
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
@@ -766,6 +769,14 @@ class RemoteDBConnection(DBConnection):
"""List server-side jobs across the database's tables."""
return LOOP.run(self._conn.list_jobs())
@override
def get_job(self, job_id: str) -> Optional["JobDescription"]:
"""Describe a single server-side job by id.
Returns None when the server has no such job.
"""
return LOOP.run(self._conn.get_job(job_id))
@override
def cancel_job(self, job_id: str) -> bool:
"""Request cancellation of a server-side job by id.
@@ -776,6 +787,14 @@ class RemoteDBConnection(DBConnection):
"""
return LOOP.run(self._conn.cancel_job(job_id))
@override
def job_history(self, job_id: Optional[str] = None) -> List[pa.RecordBatch]:
"""The lifecycle event history of a server-side job, as Arrow batches.
Lists history across all jobs when `job_id` is None.
"""
return LOOP.run(self._conn.job_history(job_id))
@override
def execute_query_async(
self,
+1 -4
View File
@@ -297,10 +297,7 @@ def test_blob_v2_projection_sources_use_typed_column_name():
def _legacy_v1_table(name):
# Legacy v1 blob columns are only writable at file version <= 2.1.
db = lancedb.connect(
"memory:///", storage_options={"new_table_data_storage_version": "2.1"}
)
db = lancedb.connect("memory:///")
schema = pa.schema(
[
pa.field("id", pa.int64()),
+1 -7
View File
@@ -193,13 +193,7 @@ class TestNamespaceConnection:
),
)
# Legacy v1 blob columns are only writable at file version <= 2.1.
table = db.create_table(
"blob_table",
data,
namespace_path=["test_ns"],
storage_options={"new_table_data_storage_version": "2.1"},
)
table = db.create_table("blob_table", data, namespace_path=["test_ns"])
df = table.to_pandas(blob_mode="lazy").sort_values("id")
blob = df["blob"].iloc[0]
+10 -38
View File
@@ -40,10 +40,6 @@ from utils import exception_output
from importlib.util import find_spec
# Legacy v1 blob columns are only writable at file version <= 2.1.
LEGACY_BLOB_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.1"}
def _blob_query_data():
return pa.table(
{
@@ -123,17 +119,13 @@ def _assert_blob_bytes_projection(df):
def _blob_query_table(db, name, blob_schema):
if blob_schema == "v1":
return db.create_table(
name, _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
)
return db.create_table(name, _blob_query_data())
return _create_blob_v2_query_table(db, name)
async def _blob_query_table_async(db, name, blob_schema):
if blob_schema == "v1":
return await db.create_table(
name, _blob_query_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
)
return await db.create_table(name, _blob_query_data())
return await _create_blob_v2_query_table_async(db, name)
@@ -283,9 +275,7 @@ async def test_query_to_pandas_kwargs(table, table_async):
def test_plain_scan_query_to_pandas_blob_modes(tmp_db, blob_mode):
pytest.importorskip("lance")
table = tmp_db.create_table(
f"test_query_to_pandas_blob_{blob_mode}",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
f"test_query_to_pandas_blob_{blob_mode}", _blob_query_data()
)
df = (
@@ -332,9 +322,7 @@ def test_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow(
):
pytest.importorskip("lance")
table = tmp_db.create_table(
"test_query_to_pandas_blob_no_arrow_collect",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
"test_query_to_pandas_blob_no_arrow_collect", _blob_query_data()
)
query = table.search().where("id = 1").select(["id", "blob"])
@@ -359,9 +347,7 @@ def test_plain_scan_query_to_pandas_blob_descriptions_flatten_uses_scanner(
):
pytest.importorskip("lance")
table = tmp_db.create_table(
"test_query_to_pandas_blob_desc_flatten",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
"test_query_to_pandas_blob_desc_flatten", _blob_query_data()
)
query = table.search().where("id = 1").select(["id", "blob"])
@@ -379,11 +365,7 @@ def test_plain_scan_query_to_pandas_blob_descriptions_flatten_uses_scanner(
def test_plain_scan_query_to_pandas_scanner_state(tmp_db):
pytest.importorskip("lance")
data = _blob_query_data()
table = tmp_db.create_table(
"test_query_to_pandas_scanner_state",
data.slice(0, 2),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
)
table = tmp_db.create_table("test_query_to_pandas_scanner_state", data.slice(0, 2))
table.add(data.slice(2, 2))
fragments = table.to_lance().get_fragments()
@@ -418,9 +400,7 @@ def test_plain_scan_query_to_pandas_scanner_state(tmp_db):
async def test_async_plain_scan_query_to_pandas_blob_projection(tmp_db_async):
pytest.importorskip("lance")
table = await tmp_db_async.create_table(
"test_async_query_to_pandas_blob_projection",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
"test_async_query_to_pandas_blob_projection", _blob_query_data()
)
lazy_df = await (
@@ -472,9 +452,7 @@ async def test_async_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow
):
pytest.importorskip("lance")
table = await tmp_db_async.create_table(
"test_async_query_to_pandas_blob_no_arrow_collect",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
"test_async_query_to_pandas_blob_no_arrow_collect", _blob_query_data()
)
query = table.query().where("id = 1").select(["id", "blob"])
@@ -496,11 +474,7 @@ async def test_async_plain_scan_query_to_pandas_blob_mode_does_not_collect_arrow
def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
pytest.importorskip("lance")
table = tmp_db.create_table(
"test_vector_query_blob_mode",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
)
table = tmp_db.create_table("test_vector_query_blob_mode", _blob_query_data())
with pytest.raises(RuntimeError, match="Lance native pandas conversion"):
table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas(
@@ -511,9 +485,7 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
def test_vector_query_to_pandas_blob_descriptions_requires_plain_scan(tmp_db):
pytest.importorskip("lance")
table = tmp_db.create_table(
"test_vector_query_blob_descriptions",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
"test_vector_query_blob_descriptions", _blob_query_data()
)
with pytest.raises(RuntimeError, match="plain scan query"):
+15 -117
View File
@@ -2467,7 +2467,7 @@ def test_remote_blob_byte_apis_not_supported_on_old_server():
def test_remote_connection_jobs_surface():
from lancedb.exceptions import JobFailedError, JobNotFoundError
from lancedb.exceptions import JobFailedError
schema = pa.schema([("state", pa.string())])
batch = pa.record_batch([pa.array(["created", "done"])], schema=schema)
@@ -2475,7 +2475,6 @@ def test_remote_connection_jobs_surface():
with pa.ipc.new_stream(sink, schema) as writer:
writer.write_batch(batch)
events_body = sink.getvalue().to_pybytes()
query_events_payloads = []
def handler(request):
content_len = int(request.headers.get("Content-Length", 0))
@@ -2513,22 +2512,6 @@ def test_remote_connection_jobs_surface():
request.end_headers()
request.wfile.write(json.dumps(rsp).encode())
elif request.path == "/v1/jobs/describe":
if payload["job_id"] == "job-2":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
dict(
job_id="job-2",
job_type="refresh_column",
job_state="DONE",
creation_ms=2000,
result=dict(rows_assigned=1000000, rows_failed=0),
)
).encode()
)
return
if payload["job_id"] != "job-1":
request.send_response(404)
request.end_headers()
@@ -2560,7 +2543,7 @@ def test_remote_connection_jobs_surface():
request.end_headers()
request.wfile.write(b'{"job_id": "job-1"}')
elif request.path == "/v1/jobs/query_events":
query_events_payloads.append(payload)
assert payload["job_id"] == "job-1"
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
request.end_headers()
@@ -2576,109 +2559,24 @@ def test_remote_connection_jobs_surface():
assert jobs[0].table == "t1"
assert jobs[1].state == "finished"
description = db.get_job("job-1")
assert description.job_type == "create_index"
assert description.state == "failed"
assert json.loads(description.spec_json) == {"column": "vec"}
assert description.failure.message == "worker died"
assert description.failure.retryable is True
assert db.get_job("missing") is None
assert db.cancel_job("job-1") is True
assert db.cancel_job("missing") is False
# Opening a job hands back a populated handle; a missing one fails.
with pytest.raises(JobNotFoundError, match="missing"):
db.open_job("missing")
finished = db.open_job("job-2")
assert finished.state == "finished"
assert finished.result == {"rows_assigned": 1000000, "rows_failed": 0}
batches = db.job_history("job-1")
assert len(batches) == 1
assert batches[0].num_rows == 2
assert batches[0].column("state").to_pylist() == ["created", "done"]
job = db.open_job("job-1")
job = db.job("job-1")
assert job.id == "job-1"
# Opening already populated the handle.
assert job.state == "failed"
assert job.spec == {"column": "vec"}
assert job.failure.message == "worker died"
assert job.status() == "failed"
with pytest.raises(JobFailedError, match="worker died"):
job.wait(timeout=timedelta(seconds=5))
def test_remote_job_handle_reports_its_own_detail():
schema = pa.schema([("state", pa.string())])
batch = pa.record_batch([pa.array(["claim_complete"])], schema=schema)
sink = pa.BufferOutputStream()
with pa.ipc.new_stream(sink, schema) as writer:
writer.write_batch(batch)
events_body = sink.getvalue().to_pybytes()
event_payloads = []
def handler(request):
content_len = int(request.headers.get("Content-Length", 0))
body = request.rfile.read(content_len) if content_len > 0 else b""
payload = json.loads(body) if body else {}
if request.path == "/v1/jobs/describe":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
dict(
job_id="job-1",
job_type="refresh_column",
job_state="DONE",
creation_ms=2000,
spec=dict(column="vec"),
result=dict(rows_assigned=1000000),
)
).encode()
)
elif request.path == "/v1/jobs/query_events":
event_payloads.append(payload)
request.send_response(200)
request.send_header("Content-Type", "application/vnd.apache.arrow.stream")
request.end_headers()
request.wfile.write(events_body)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
job = db.open_job("job-1")
# Opening populates the handle in the same round trip.
assert job.state == "finished"
job.refresh()
assert job.job_type == "refresh_column"
assert job.creation_ms == 2000
assert job.spec == {"column": "vec"}
assert job.result == {"rows_assigned": 1000000}
assert job.failure is None
# The JSON payloads stay reachable, but as internal APIs.
assert json.loads(job._spec_json) == {"column": "vec"}
assert json.loads(job._result_json) == {"rows_assigned": 1000000}
# print() shows everything the handle knows and nothing it does not.
# print() lays every known field out on its own line, with the JSON
# payloads indented rather than crammed onto one line.
assert repr(job) == "\n".join(
[
"Job(",
" id='job-1',",
" state='finished',",
" job_type='refresh_column',",
" creation_ms=2000,",
" spec={",
' "column": "vec"',
" },",
" result={",
' "rows_assigned": 1000000',
" },",
")",
]
)
# Nothing it does not know shows up.
assert "failure" not in repr(job)
events = job.events(filter="state = 'claim_complete'", limit=500)
assert isinstance(events, pa.Table)
assert events.column("state").to_pylist() == ["claim_complete"]
# The handle supplies job_id; the caller only narrows the query.
assert event_payloads[-1] == {
"job_id": "job-1",
"limit": 500,
"filter": "state = 'claim_complete'",
}
+4 -16
View File
@@ -64,23 +64,15 @@ async def _blob_v2_table_async(db: AsyncConnection, name: str):
return table
# Legacy v1 blob columns are only writable at file version <= 2.1.
LEGACY_BLOB_STORAGE_OPTIONS = {"new_table_data_storage_version": "2.1"}
def _blob_table(db: DBConnection, name: str, blob_schema: str):
if blob_schema == "v1":
return db.create_table(
name, data=_blob_test_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
)
return db.create_table(name, data=_blob_test_data())
return _blob_v2_table(db, name)
async def _blob_table_async(db: AsyncConnection, name: str, blob_schema: str):
if blob_schema == "v1":
return await db.create_table(
name, data=_blob_test_data(), storage_options=LEGACY_BLOB_STORAGE_OPTIONS
)
return await db.create_table(name, data=_blob_test_data())
return await _blob_v2_table_async(db, name)
@@ -155,11 +147,7 @@ def test_table_to_pandas_invalid_blob_mode_non_blob_table(tmp_db: DBConnection):
@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"])
def test_table_to_pandas_blob_modes(tmp_db: DBConnection, blob_mode):
pytest.importorskip("lance")
table = tmp_db.create_table(
f"test_to_pandas_blob_{blob_mode}",
_blob_test_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
)
table = tmp_db.create_table(f"test_to_pandas_blob_{blob_mode}", _blob_test_data())
df = table.to_pandas(blob_mode=blob_mode)
@@ -3971,7 +3959,7 @@ def test_stats(mem_db: DBConnection):
print(f"{stats=}")
assert stats == {
# Full on-disk size of the data file, footer and metadata included.
"total_bytes": 637,
"total_bytes": 633,
"num_rows": 2,
"num_indices": 0,
"fragment_stats": {
+35 -8
View File
@@ -13,7 +13,11 @@ use crate::{
runtime::future_into_py,
table::Table,
};
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
use arrow::{
datatypes::Schema,
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, ToPyArrow},
};
use lancedb::{
connection::Connection as LanceConnection,
connection::NamespaceClientPushdownOperation,
@@ -24,7 +28,7 @@ use pyo3::{
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
exceptions::{PyRuntimeError, PyValueError},
pyclass, pyfunction, pymethods,
types::{PyAnyMethods, PyDict, PyDictMethods, PyList},
types::{PyAnyMethods, PyDict, PyDictMethods, PyList, PyListMethods},
};
#[pyclass]
@@ -640,12 +644,9 @@ impl Connection {
})
}
pub fn open_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let job = inner.open_job(&job_id).await.infer_error()?;
Ok(crate::job::Job::new(job))
})
pub fn job(&self, job_id: String) -> PyResult<crate::job::Job> {
let inner = self.get_inner()?.clone();
Ok(crate::job::Job::new(inner.job(job_id).infer_error()?))
}
pub fn create_function_async(
@@ -715,12 +716,38 @@ impl Connection {
})
}
pub fn get_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let description = inner.get_job(&job_id).await.infer_error()?;
Ok(description.map(crate::job::JobDescription::from))
})
}
pub fn cancel_job(self_: PyRef<'_, Self>, job_id: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.cancel_job(&job_id).await.infer_error()
})
}
#[pyo3(signature = (job_id=None))]
pub fn job_history(
self_: PyRef<'_, Self>,
job_id: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let batches = inner.job_history(job_id.as_deref()).await.infer_error()?;
Python::attach(|py| {
let list = PyList::empty(py);
for batch in batches {
list.append(batch.to_pyarrow(py)?)?;
}
Ok(list.unbind())
})
})
}
}
#[pyfunction]
-6
View File
@@ -114,12 +114,6 @@ impl<T> PythonErrorExt<T> for std::result::Result<T, LanceError> {
.getattr(intern!(py, "JobCancelledError"))?;
Err(PyErr::from_value(cls.call1((err.to_string(),))?))
}),
LanceError::JobNotFound { .. } => Python::attach(|py| {
let cls = py
.import(intern!(py, "lancedb.exceptions"))?
.getattr(intern!(py, "JobNotFoundError"))?;
Err(PyErr::from_value(cls.call1((err.to_string(),))?))
}),
_ => self.runtime_error(),
},
}
+9 -126
View File
@@ -4,50 +4,11 @@
use std::sync::Arc;
use crate::runtime::future_into_py;
use arrow::{
datatypes::Schema,
pyarrow::{IntoPyArrow, Table as PyArrowTable},
};
use lancedb::job::JobEventsRequest;
use pyo3::{
Bound, PyAny, PyRef, PyResult, Python,
exceptions::PyValueError,
pyclass, pymethods,
types::{PyAnyMethods, PyDict, PyDictMethods},
};
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
use serde::Serialize;
use crate::error::PythonErrorExt;
const REPR_INDENT: &str = " ";
/// Parse a stored JSON payload into Python data. The bindings carry these as
/// strings because that is what crosses the boundary cheaply; the public
/// Python surface is the parsed form.
fn parse_json_payload<'py>(
py: Python<'py>,
raw: Option<&str>,
) -> PyResult<Option<Bound<'py, PyAny>>> {
match raw {
None => Ok(None),
Some(raw) => Ok(Some(py.import("json")?.call_method1("loads", (raw,))?)),
}
}
/// A payload rendered as indented JSON, aligned under the field that holds it.
fn pretty_json_payload(py: Python<'_>, raw: Option<&str>) -> PyResult<Option<String>> {
let Some(parsed) = parse_json_payload(py, raw)? else {
return Ok(None);
};
let kwargs = PyDict::new(py);
kwargs.set_item("indent", 4)?;
let rendered: String = py
.import("json")?
.call_method("dumps", (parsed,), Some(&kwargs))?
.extract()?;
Ok(Some(rendered.replace('\n', &format!("\n{REPR_INDENT}"))))
}
#[pyclass]
pub struct Job {
inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
@@ -106,48 +67,6 @@ impl Job {
Ok(())
})
}
pub fn refresh(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.refresh().await.infer_error()?;
Ok(())
})
}
/// The last observed lifecycle state, without contacting the backend.
#[getter]
pub fn _state(&self) -> Option<String> {
self.inner.state()
}
/// The last observed server-side record. `None` for an in-process job.
#[getter]
pub fn _description(&self) -> Option<JobDescription> {
self.inner.description().map(JobDescription::from)
}
#[pyo3(signature = (*, limit=None, filter=None))]
pub fn events(
self_: PyRef<'_, Self>,
limit: Option<u32>,
filter: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
let request = JobEventsRequest { limit, filter };
future_into_py(self_.py(), async move {
let batches = inner.events(request).await.infer_error()?;
Python::attach(|py| {
let schema = batches
.first()
.map(|batch| batch.schema())
.unwrap_or_else(|| Arc::new(Schema::empty()));
let table = PyArrowTable::try_new(batches, schema)
.map_err(|err| PyValueError::new_err(err.to_string()))?;
table.into_pyarrow(py).map(|table| table.unbind())
})
})
}
}
/// A row from `Connection.list_jobs`: one server-side job.
@@ -202,7 +121,7 @@ impl JobFailureInfo {
}
}
/// The server-side record behind a `Job` handle.
/// A described job from `Connection.get_job`.
#[pyclass(get_all, skip_from_py_object)]
#[derive(Clone)]
pub struct JobDescription {
@@ -210,49 +129,17 @@ pub struct JobDescription {
job_type: String,
state: String,
creation_ms: i64,
/// Internal: the wire form behind the `spec` property.
_spec_json: Option<String>,
/// Internal: the wire form behind the `result` property.
_result_json: Option<String>,
spec_json: Option<String>,
failure: Option<JobFailureInfo>,
}
#[pymethods]
impl JobDescription {
/// The job-type-specific specification it was submitted with.
#[getter]
fn spec<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
parse_json_payload(py, self._spec_json.as_deref())
}
/// The job-type-specific terminal result. `None` until the job succeeds.
#[getter]
fn result<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyAny>>> {
parse_json_payload(py, self._result_json.as_deref())
}
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
let mut fields = vec![
format!("job_id={:?}", self.job_id),
format!("job_type={:?}", self.job_type),
format!("state={:?}", self.state),
format!("creation_ms={}", self.creation_ms),
];
// Lay the payloads out as indented JSON, the same way the `Job` repr
// does, so the two agree on how the same data looks.
for (name, payload) in [("spec", &self._spec_json), ("result", &self._result_json)] {
if let Some(rendered) = pretty_json_payload(py, payload.as_deref())? {
fields.push(format!("{name}={rendered}"));
}
}
if let Some(failure) = &self.failure {
fields.push(format!("failure={}", failure.__repr__()));
}
let body = fields
.iter()
.map(|field| format!("\n{REPR_INDENT}{field},"))
.collect::<String>();
Ok(format!("JobDescription({body}\n)"))
fn __repr__(&self) -> String {
format!(
"JobDescription(job_id={:?}, job_type={:?}, state={:?}, creation_ms={})",
self.job_id, self.job_type, self.state, self.creation_ms
)
}
}
@@ -263,11 +150,7 @@ impl From<lancedb::database::JobDescription> for JobDescription {
job_type: description.job_type,
state: description.state,
creation_ms: description.creation_ms,
_spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
_result_json: description
.result
.filter(|result| !result.is_null())
.map(|result| result.to_string()),
spec_json: (!description.spec.is_null()).then(|| description.spec.to_string()),
failure: description.failure.map(|failure| JobFailureInfo {
phase: failure.phase,
message: failure.message,
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.39.0-beta.6"
version = "0.39.0-beta.1"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
+8 -10
View File
@@ -532,11 +532,10 @@ mod tests {
fn storage_version_bumps_to_v2_2() {
let mut params = WriteParams::default();
ensure_blob_storage_version(&blob_schema(), &mut params);
let resolved = params
.data_storage_version
.unwrap_or(LanceFileVersion::Stable)
.resolve();
assert_eq!(resolved, ConcreteFileVersion::V2_2);
assert_eq!(
params.data_storage_version.unwrap().resolve(),
ConcreteFileVersion::V2_2
);
assert!(!params.enable_stable_row_ids);
}
@@ -548,11 +547,10 @@ mod tests {
};
ensure_blob_storage_version(&blob_schema(), &mut params);
assert!(params.enable_stable_row_ids);
let resolved = params
.data_storage_version
.unwrap_or(LanceFileVersion::Stable)
.resolve();
assert_eq!(resolved, ConcreteFileVersion::V2_2);
assert_eq!(
params.data_storage_version.unwrap().resolve(),
ConcreteFileVersion::V2_2
);
}
#[test]
+21 -29
View File
@@ -23,8 +23,8 @@ use crate::connection::create_table::CreateTableBuilder;
use crate::data::scannable::Scannable;
use crate::database::listing::ListingDatabase;
use crate::database::{
CloneTableRequest, Database, DatabaseOptions, JobInfo, OpenTableRequest, ReadConsistency,
TableNamesRequest,
CloneTableRequest, Database, DatabaseOptions, JobDescription, JobInfo, OpenTableRequest,
ReadConsistency, TableNamesRequest,
};
use crate::embeddings::{EmbeddingRegistry, MemoryRegistry};
use crate::error::{Error, Result};
@@ -670,34 +670,14 @@ impl Connection {
self.internal.read_consistency().await
}
/// Open a server-side job by id, returning a handle with its record
/// already populated. Fails with [`crate::Error::JobNotFound`] when the
/// server has no such job, the way [`Connection::open_table`] does for a
/// missing table.
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
/// waiting on or cancelling the job.
///
/// This is the one way in: the returned [`crate::job::Job`] answers for
/// its own state, specification, result, failure and event history, so
/// there is no separate connection-level call for any of them.
///
/// # Example
///
/// ```no_run
/// # use lancedb::job::JobEventsRequest;
/// # async fn open_job(
/// # connection: &lancedb::Connection,
/// # job_id: &str,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// let job = connection.open_job(job_id).await?;
/// println!("{:?} {:?}", job.state(), job.result());
/// let done = job
/// .events(JobEventsRequest::default().filter("state = 'claim_complete'"))
/// .await?;
/// println!("{} completions", done.iter().map(|b| b.num_rows()).sum::<usize>());
/// # Ok(())
/// # }
/// ```
pub async fn open_job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
self.internal.open_job(job_id.as_ref()).await
/// The handle is constructed without a server round trip; an unknown id
/// surfaces when the handle is used. Only server-backed databases support
/// job handles by id.
pub fn job(&self, job_id: impl AsRef<str>) -> Result<crate::job::Job> {
self.internal.job(job_id.as_ref())
}
/// List server-side jobs across the database's tables.
@@ -705,12 +685,24 @@ impl Connection {
self.internal.list_jobs().await
}
/// Describe a single server-side job by id. `None` when the server has no
/// such job.
pub async fn get_job(&self, job_id: impl AsRef<str>) -> Result<Option<JobDescription>> {
self.internal.get_job(job_id.as_ref()).await
}
/// Request cancellation of a server-side job by id. Returns true if the
/// server accepted the cancellation, false if no such job exists.
pub async fn cancel_job(&self, job_id: impl AsRef<str>) -> Result<bool> {
self.internal.cancel_job(job_id.as_ref()).await
}
/// The lifecycle event history of a server-side job (all jobs when
/// `job_id` is `None`), as recorded Arrow batches.
pub async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
self.internal.job_history(job_id).await
}
/// Drop a table in the database.
///
/// # Arguments
+20 -13
View File
@@ -18,6 +18,8 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use arrow_array::RecordBatch;
use lance::dataset::ReadParams;
use lance_namespace::LanceNamespace;
use lance_namespace::models::{
@@ -204,8 +206,8 @@ pub enum ReadConsistency {
/// compaction, column refresh, ...).
#[derive(Debug, Clone)]
pub struct JobInfo {
/// The job id -- what [`Database::open_job`] and
/// [`Database::cancel_job`] accept.
/// The job id -- what [`Database::get_job`] and [`Database::cancel_job`]
/// accept.
pub job_id: String,
/// The table the job runs against, without URI or namespace.
pub table: String,
@@ -216,8 +218,8 @@ pub struct JobInfo {
pub created_at_millis: i64,
}
/// The server-side record behind a [`crate::job::Job`] handle: lifecycle
/// state plus the job-type-specific specification and result.
/// A described job from [`Database::get_job`]: lifecycle state plus the
/// job-type-specific specification.
#[derive(Debug, Clone)]
pub struct JobDescription {
pub job_id: String,
@@ -228,10 +230,6 @@ pub struct JobDescription {
pub creation_ms: i64,
/// The job-type-specific specification. Null when the server omits it.
pub spec: serde_json::Value,
/// The job-type-specific terminal result, for job types that define one.
/// `None` until the job succeeds, so a job that never terminates reports
/// its progress through [`crate::job::Job::events`] instead.
pub result: Option<serde_json::Value>,
/// Why the job failed, when the job is failed and the server reports a
/// reason.
pub failure: Option<crate::error::JobFailure>,
@@ -317,22 +315,31 @@ pub trait Database:
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
/// Open a job by id, returning a handle with its record already
/// populated. Fails with [`crate::Error::JobNotFound`] when the server has
/// no such job.
async fn open_job(&self, _job_id: &str) -> Result<crate::job::Job> {
job_op_not_supported("open_job")
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
/// waiting on or cancelling the job. The handle is constructed without a
/// server round trip; an unknown id surfaces when the handle is used.
fn job(&self, _job_id: &str) -> Result<crate::job::Job> {
job_op_not_supported("job")
}
/// List server-side jobs across the database's tables.
async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
job_op_not_supported("list_jobs")
}
/// Describe a single job by id. `None` when the server has no such job.
async fn get_job(&self, _job_id: &str) -> Result<Option<JobDescription>> {
job_op_not_supported("get_job")
}
/// Request cancellation of a job by id. Returns true if the server
/// accepted the cancellation, false if no such job exists. Cancelling an
/// already-terminal job is a no-op success.
async fn cancel_job(&self, _job_id: &str) -> Result<bool> {
job_op_not_supported("cancel_job")
}
/// The lifecycle event history of a job (all jobs when `job_id` is
/// `None`), as recorded Arrow batches.
async fn job_history(&self, _job_id: Option<&str>) -> Result<Vec<RecordBatch>> {
job_op_not_supported("job_history")
}
/// Start executing a SQL statement on a remote database.
async fn execute_query_async(
&self,
-5
View File
@@ -3,7 +3,6 @@
//! Namespace-based database implementation that delegates table management to lance-namespace
use lance_datafusion::utils::StreamingWriteSource;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, Mutex};
@@ -305,10 +304,6 @@ impl Database for LanceNamespaceDatabase {
}
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
// Refuse a bad declaration before the namespace records a table.
crate::table::computed_columns::ensure_declarations_are_planned(
&request.data.arrow_schema(),
)?;
let mut table_id = request.namespace_path.clone();
table_id.push(request.name.clone());
let mut existing_table = None;
-2
View File
@@ -102,8 +102,6 @@ pub enum Error {
},
#[snafu(display("Job{} was cancelled", job_id.as_ref().map(|id| format!(" {id}")).unwrap_or_default()))]
JobCancelled { job_id: Option<String> },
#[snafu(display("Job '{job_id}' was not found"))]
JobNotFound { job_id: String },
// 3rd party / external errors
#[snafu(display("object_store error: {source}"))]
+12 -247
View File
@@ -3,53 +3,16 @@
//! Handles to operations a server may run asynchronously.
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::sync::Arc;
use arrow_array::RecordBatch;
use async_trait::async_trait;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use tokio::sync::watch;
use tokio::task::{AbortHandle, JoinHandle};
use crate::database::JobDescription;
use crate::error::{Error, JobFailure, Result};
/// Which of a job's events [`Job::events`] returns.
///
/// The handle already knows which job to ask about, so this narrows the
/// query rather than naming one.
#[derive(Debug, Clone, Default)]
pub struct JobEventsRequest {
/// Maximum event rows to return. The server applies its own default
/// (1000 rows) and maximum (10,000 rows) when this is `None`, and
/// truncates without saying so, which matters for a job with one event
/// per fragment.
pub limit: Option<u32>,
/// SQL-like filter over the event columns `state`, `updated_by`,
/// `emitted_from`, `emitted_by`, and `claim_entity`. For example
/// `state = 'claim_complete'` selects only per-claim completions.
pub filter: Option<String>,
}
impl JobEventsRequest {
pub fn limit(mut self, limit: u32) -> Self {
self.limit = Some(limit);
self
}
pub fn filter(mut self, filter: impl Into<String>) -> Self {
self.filter = Some(filter.into());
self
}
}
fn job_detail_not_supported<T>(what: &str) -> Result<T> {
Err(Error::NotSupported {
message: format!("{what} is only available for server-side jobs"),
})
}
/// Backend-specific tracking for an asynchronous operation.
#[async_trait]
pub(crate) trait JobHandle: Send + Sync {
@@ -60,15 +23,6 @@ pub(crate) trait JobHandle: Send + Sync {
async fn status(&self) -> Result<String>;
async fn wait(&self) -> Result<TerminalResult>;
async fn cancel(&self) -> Result<()>;
/// The job's full server-side record. Backends that run the operation in
/// this process have none and keep the default.
async fn describe(&self) -> Result<JobDescription> {
job_detail_not_supported("describing a job")
}
/// The job's recorded lifecycle events.
async fn events(&self, _request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
job_detail_not_supported("job event history")
}
}
/// A backend-neutral successful terminal result.
@@ -131,34 +85,16 @@ enum JobInner<T> {
Completed(T),
}
/// What a handle last learned about its job. `state` is separate because an
/// in-process job can report one but has no server-side record behind it.
#[derive(Default)]
struct JobCache {
state: Option<String>,
description: Option<JobDescription>,
}
/// A handle to an operation that may still be running.
///
/// The operation may already be complete when the handle is created. `T` is
/// the endpoint's successful terminal result; unit-result operations use the
/// default `Job<()>`.
///
/// The detail accessors ([`Job::state`], [`Job::job_type`], ...) read what the
/// handle last observed. Submitting an operation returns only a job id, so
/// populating them eagerly would cost an extra round trip on every call:
///
/// - [`Job::refresh`] and [`Job::status`] fetch the whole record.
/// - [`Job::wait`] records the terminal state it establishes, but not the rest
/// of the record; call [`Job::refresh`] for that.
/// - Everything is `None` until one of those runs.
pub struct Job<T = ()>
where
T: Clone + Send + Sync + 'static,
{
inner: JobInner<T>,
cache: RwLock<JobCache>,
}
impl<T> std::fmt::Debug for Job<T>
@@ -166,40 +102,18 @@ where
T: Clone + Send + Sync + 'static,
{
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let cache = self.cache_read();
let mut out = f.debug_struct("Job");
out.field("id", &self.id())
.field("done", &matches!(self.inner, JobInner::Completed(_)));
if let Some(state) = &cache.state {
out.field("state", state);
}
if let Some(description) = &cache.description {
out.field("job_type", &description.job_type)
.field("creation_ms", &description.creation_ms);
if !description.spec.is_null() {
out.field("spec", &description.spec);
}
if let Some(result) = &description.result {
out.field("result", result);
}
if let Some(failure) = &description.failure {
out.field("failure", failure);
}
}
out.finish()
f.debug_struct("Job")
.field("id", &self.id())
.field("done", &matches!(self.inner, JobInner::Completed(_)))
.finish()
}
}
impl Job<()> {
/// A job whose operation finished before the handle was created. Its
/// state is known without asking anyone, so the cache starts populated.
/// A job whose operation finished before the handle was created.
pub(crate) fn new_done() -> Self {
Self {
inner: JobInner::Completed(()),
cache: RwLock::new(JobCache {
state: Some("finished".to_string()),
description: None,
}),
}
}
@@ -209,21 +123,8 @@ impl Job<()> {
handle,
decode: Arc::new(|_| Ok(())),
},
cache: RwLock::default(),
}
}
/// A handle whose record the caller has already fetched, so the detail
/// accessors answer without a second round trip.
pub(crate) fn opened(handle: Box<dyn JobHandle>, description: JobDescription) -> Self {
let job = Self::new(handle);
{
let mut cache = job.cache_write();
cache.state = Some(description.state.clone());
cache.description = Some(description);
}
job
}
}
impl<T> Job<T>
@@ -237,7 +138,6 @@ where
handle,
decode: Arc::new(TerminalResult::decode::<T>),
},
cache: RwLock::default(),
}
}
}
@@ -269,124 +169,16 @@ where
}
}
fn cache_read(&self) -> RwLockReadGuard<'_, JobCache> {
self.cache.read().unwrap_or_else(|err| err.into_inner())
}
fn cache_write(&self) -> RwLockWriteGuard<'_, JobCache> {
self.cache.write().unwrap_or_else(|err| err.into_inner())
}
/// Asks the backend for this job's current state, and for a server-side
/// job its full record, then caches the answer for the detail accessors.
///
/// In-process operations have no server-side record, so only
/// [`Job::state`] is populated for them.
pub async fn refresh(&self) -> Result<()> {
self.refresh_state().await.map(|_| ())
}
/// Refreshes and reports the state, which every backend can answer.
async fn refresh_state(&self) -> Result<String> {
let JobInner::Handle { handle, .. } = &self.inner else {
let state = "finished".to_string();
self.cache_write().state = Some(state.clone());
return Ok(state);
};
match handle.describe().await {
Ok(description) => {
let state = description.state.clone();
let mut cache = self.cache_write();
cache.state = Some(state.clone());
cache.description = Some(description);
Ok(state)
}
// An in-process job knows its own state and nothing more.
Err(Error::NotSupported { .. }) => {
let state = handle.status().await?;
self.cache_write().state = Some(state.clone());
Ok(state)
}
Err(err) => Err(err),
}
}
/// The operation's current lifecycle state: "running", "finished",
/// "failed", or "cancelled".
///
/// A point snapshot; unlike [`Job::wait`] it does not block, raise on a
/// terminal failure state, or retry. States a newer server reports that
/// this client version does not know pass through as-is. Also refreshes
/// the detail accessors.
/// this client version does not know pass through as-is.
pub async fn status(&self) -> Result<String> {
self.refresh_state().await
}
/// The last lifecycle state this handle observed, without contacting the
/// backend. `None` until the handle has.
pub fn state(&self) -> Option<String> {
self.cache_read().state.clone()
}
/// The whole server-side record this handle last observed. The accessors
/// below read individual fields out of it. `None` for an in-process job,
/// which has no such record.
pub fn description(&self) -> Option<JobDescription> {
self.cache_read().description.clone()
}
/// The job's type, as the server names it. `None` for an in-process job.
pub fn job_type(&self) -> Option<String> {
self.with_description(|description| description.job_type.clone())
}
/// When the job was created, in milliseconds since the epoch. `None` for
/// an in-process job.
pub fn creation_ms(&self) -> Option<i64> {
self.with_description(|description| description.creation_ms)
}
/// The job-type-specific specification it was submitted with.
pub fn spec(&self) -> Option<Value> {
self.with_description(|description| description.spec.clone())
.filter(|spec| !spec.is_null())
}
/// The job-type-specific terminal result, as reported data rather than the
/// typed model [`Job::wait`] returns. `None` until the job succeeds.
pub fn result(&self) -> Option<Value> {
self.with_description(|description| description.result.clone())
.flatten()
}
/// Why the job failed, when it failed and the server reports a reason.
pub fn failure(&self) -> Option<JobFailure> {
self.with_description(|description| description.failure.clone())
.flatten()
}
fn with_description<R>(&self, read: impl FnOnce(&JobDescription) -> R) -> Option<R> {
self.cache_read().description.as_ref().map(read)
}
/// This job's recorded lifecycle events.
///
/// Unlike the detail accessors, which report a terminal result only once
/// the job reaches one, events are written as the job runs and outlive the
/// workers that produced them. A distributed job records a
/// `claim`/`claim_complete` pair per unit of work, each carrying
/// `rows_processed`, so a job that never finishes still accounts for what
/// it did. In-process operations keep no event history.
pub async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
match &self.inner {
JobInner::Handle { handle, .. } => handle.events(request).await,
// The operation finished before the handle existed, so there is no
// id to query with even when a server ran it.
JobInner::Completed(_) => Err(Error::NotSupported {
message: "this operation completed before its handle was created, so it \
carries no job id to query events with"
.to_string(),
}),
JobInner::Handle { handle, .. } => handle.status().await,
JobInner::Completed(_) => Ok("finished".to_string()),
}
}
@@ -398,19 +190,8 @@ where
/// [`crate::Error::JobCancelled`] if it was cancelled.
pub async fn wait(&self) -> Result<T> {
match &self.inner {
JobInner::Handle { handle, decode } => {
let settled = handle.wait().await;
// Waiting already established a terminal state; record it so
// the detail accessors do not need another round trip for it.
if let Some(state) = terminal_state(&settled) {
self.cache_write().state = Some(state.to_string());
}
(decode)(settled?)
}
JobInner::Completed(result) => {
self.cache_write().state = Some("finished".to_string());
Ok(result.clone())
}
JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
JobInner::Completed(result) => Ok(result.clone()),
}
}
@@ -443,36 +224,20 @@ where
U: Clone + Send + Sync + 'static,
F: Fn(T) -> U + Send + Sync + 'static,
{
// The mapped handle tracks the same job, so it inherits what this one
// has already learned about it.
let Self { inner, cache } = self;
match inner {
match self.inner {
JobInner::Handle { handle, decode } => Job {
inner: JobInner::Handle {
handle,
decode: Arc::new(move |result| Ok(map((decode)(result)?))),
},
cache,
},
JobInner::Completed(result) => Job {
inner: JobInner::Completed(map(result)),
cache,
},
}
}
}
/// The lifecycle state a settled [`JobHandle::wait`] implies.
fn terminal_state(settled: &Result<TerminalResult>) -> Option<&'static str> {
match settled {
Ok(_) => Some("finished"),
Err(Error::JobFailed { .. }) => Some("failed"),
Err(Error::JobCancelled { .. }) => Some("cancelled"),
// Anything else is a transport failure, not a verdict on the job.
Err(_) => None,
}
}
/// How an in-process operation ended. Cloneable so every waiter can be given
/// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
#[derive(Clone)]
File diff suppressed because it is too large Load Diff
+17 -536
View File
@@ -24,8 +24,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
use arrow_array::cast::AsArray;
use arrow_array::types::UInt64Type;
use arrow_array::{RecordBatch, UInt64Array, new_null_array};
use arrow_schema::{FieldRef, Schema as ArrowSchema, SchemaRef};
use arrow_array::{RecordBatch, UInt64Array};
use arrow_schema::{Schema as ArrowSchema, SchemaRef};
use datafusion::common::ScalarValue;
use datafusion::error::DataFusionError;
use datafusion::physical_plan::SendableRecordBatchStream;
@@ -34,7 +34,7 @@ use datafusion::prelude::{col, lit};
use futures::{StreamExt, TryStreamExt};
use lance::Dataset;
use lance::dataset::mem_wal::DatasetMemWalExt;
use lance::dataset::transaction::{Operation, Transaction, UpdateMode};
use lance::dataset::transaction::{Operation, Transaction};
use lance::dataset::write::delete::DeleteBuilder;
use lance::dataset::write::merge_insert::inserted_rows::{
KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue,
@@ -51,9 +51,6 @@ use super::{
definition_to_metadata,
};
use crate::database::OpenTableRequest;
use crate::table::computed_columns::{
computed_column_from_field, computed_columns, ensure_declarations_are_planned,
};
use crate::table::{NativeTable, NativeTableExt, Table};
use crate::{Error, Result};
@@ -170,52 +167,30 @@ pub(crate) async fn execute_refresh(
.map(|p| (p.output.clone(), p.expression.clone()))
.collect();
validate_inputs(&source_ds, definition)?;
let (replanned, planned_fields, _renames) = super::plan(
let (replanned, mut planned_fields, _renames) = super::plan(
source_schema,
&definition.source_table,
&definition.source_namespace,
Some(&projections),
&projections,
definition.filter.as_deref(),
definition.limit,
)?;
let mut planned_fields = planned_fields;
planned_fields.push(arrow_schema::Field::new(
SOURCE_ROW_ID_COLUMN,
arrow_schema::DataType::UInt64,
false,
));
// A computed column is not planned from the source: refresh writes it
// NULL and its declaration's owner fills it. Its declaration must still
// be complete, and it must be able to hold NULL.
let physical = ArrowSchema::from(view_ds.schema());
let mut computed = computed_columns(&physical).into_iter().map(|c| c.name);
if let Some(name) = computed.by_ref().find(|name| {
physical
.field_with_name(name)
.is_ok_and(|f| !f.is_nullable())
}) {
return Err(Error::Schema {
message: format!(
"computed column '{name}' of view '{}' cannot hold NULL; recreate the view",
view.name()
),
});
}
ensure_declarations_are_planned(&physical)?;
let physical_planned: Vec<&FieldRef> = physical
let planned_shape: Vec<_> = planned_fields
.iter()
.map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
.collect();
let physical_shape: Vec<_> = physical
.fields()
.iter()
.filter(|f| computed_column_from_field(f).is_none())
.map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable()))
.collect();
// A projected column that became nullable at the source still fits the
// view's nullable field; the reverse would not.
let matches = planned_fields.len() == physical_planned.len()
&& planned_fields.iter().zip(&physical_planned).all(|(e, p)| {
e.name() == p.name()
&& e.data_type() == p.data_type()
&& (p.is_nullable() || !e.is_nullable())
});
if !matches {
if planned_shape != physical_shape {
return Err(Error::Schema {
message: format!(
"the stored definition of view '{}' does not produce this \
@@ -254,18 +229,11 @@ pub(crate) async fn execute_refresh(
.get(SOURCE_VERSION_TS_META_KEY)
.and_then(|raw| raw.parse().ok());
// The watermark speaks only for the view state its refresh left behind;
// any other commit on the view since then is drift, except a fill of its
// computed columns, which rewrites nothing refresh certifies.
let recorded_view_version = metadata
// any other commit on the view since then is drift.
let view_intact = metadata
.get(VIEW_VERSION_META_KEY)
.and_then(|raw| raw.parse::<u64>().ok());
let view_intact = match recorded_view_version {
Some(recorded) if recorded == view_ds.version().version => true,
Some(recorded) if recorded < view_ds.version().version => {
only_computed_rewrites_since(&view_ds, recorded).await?
}
_ => false,
};
.and_then(|raw| raw.parse::<u64>().ok())
== Some(view_ds.version().version);
if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) {
return Ok(RefreshMaterializedViewResult {
@@ -1122,69 +1090,6 @@ struct RowScope {
limit: Option<u64>,
}
/// Whether every commit on the view after `recorded` is a fill of its
/// computed columns: a column rewrite or data replacement touching only
/// those fields and neither adding nor removing rows. A version whose
/// transaction cannot be read is not proven, so it counts as drift.
async fn only_computed_rewrites_since(view_ds: &Dataset, recorded: u64) -> Result<bool> {
// A fill may write any field under a computed column, so the whole
// subtree counts, not only the root.
let physical = ArrowSchema::from(view_ds.schema());
fn subtree(field: &lance_core::datatypes::Field, ids: &mut Vec<u32>) {
ids.push(field.id as u32);
for child in &field.children {
subtree(child, ids);
}
}
let mut computed_fields = Vec::new();
for column in computed_columns(&physical) {
if let Some(field) = view_ds.schema().field(&column.name) {
subtree(field, &mut computed_fields);
}
}
if computed_fields.is_empty() {
return Ok(false);
}
for version in recorded + 1..=view_ds.version().version {
let Some(transaction) = view_ds.read_transaction_by_version(version).await? else {
return Ok(false);
};
let fill = match &transaction.operation {
Operation::Update {
removed_fragment_ids,
new_fragments,
fields_modified,
update_mode: Some(UpdateMode::RewriteColumns),
..
} => {
removed_fragment_ids.is_empty()
&& new_fragments.is_empty()
&& !fields_modified.is_empty()
&& fields_modified
.iter()
.all(|field| computed_fields.contains(field))
}
// What `refresh_column` commits for a SQL declaration.
Operation::DataReplacement { replacements } => {
!replacements.is_empty()
&& replacements.iter().all(|group| {
!group.1.fields.is_empty()
&& group
.1
.fields
.iter()
.all(|field| computed_fields.contains(&(*field as u32)))
})
}
_ => false,
};
if !fill {
return Ok(false);
}
}
Ok(true)
}
async fn compute_stream(
source: &Dataset,
definition: &MaterializedViewDefinition,
@@ -1253,10 +1158,6 @@ async fn compute_stream(
let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?;
let mut columns = Vec::with_capacity(out_schema.fields().len());
for field in out_schema.fields() {
if computed_column_from_field(field).is_some() {
columns.push(new_null_array(field.data_type(), batch.num_rows()));
continue;
}
let name = if field.name() == SOURCE_ROW_ID_COLUMN {
ROW_ID
} else {
@@ -2867,7 +2768,7 @@ mod tests {
let (conn, source) = db_with_source(vec![1]).await;
let prepared = crate::materialized_view::prepare_declaration(
&source,
Some(&[("x".into(), "x".into()), ("twice".into(), "x * 2".into())]),
&[("x".into(), "x".into()), ("twice".into(), "x * 2".into())],
None,
None,
)
@@ -3231,424 +3132,4 @@ mod tests {
let err = view.refresh().execute().await.unwrap_err();
assert!(err.to_string().contains("source table 'src'"), "{err}");
}
/// A view with a computed column, declared over `people` and refreshed.
async fn refreshed_computed_view(conn: &Connection) -> MaterializedView {
use crate::materialized_view::tests::{computed_field, people, test_binding};
let source = people(conn).await;
let view = crate::materialized_view::prepare_declaration(
&source,
Some(&[
("id".to_string(), "id".to_string()),
("name".to_string(), "name".to_string()),
]),
None,
None,
)
.await
.unwrap()
.with_computed_columns(
vec![(2, computed_field("emb", "fb_1", "name"))],
&[test_binding("fb_1", "name", "emb")],
)
.unwrap()
.create("v")
.await
.unwrap();
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Rebuild);
view
}
async fn unfilled(view: &MaterializedView) -> usize {
view.table()
.count_rows(Some("emb IS NULL".to_string()))
.await
.unwrap()
}
async fn append_people(conn: &Connection, ids: Vec<i32>, names: Vec<&str>) {
let batch = record_batch!(("id", Int32, ids), ("name", Utf8, names)).unwrap();
conn.open_table("people")
.execute()
.await
.unwrap()
.add(batch)
.execute()
.await
.unwrap();
}
/// Commit the fill job's shape on the view: a column rewrite of
/// `fields`, touching no rows. The data is left as it is; what matters
/// here is how the next refresh classifies the commit.
async fn commit_column_rewrite(view: &MaterializedView, fields: &[&str]) {
let native = view.table().as_native().unwrap();
native.dataset.reload().await.unwrap();
let dataset = native.dataset.get().await.unwrap().as_ref().clone();
let fields_modified = fields
.iter()
.map(|name| dataset.schema().field(name).unwrap().id as u32)
.collect();
let updated_fragments = dataset
.get_fragments()
.iter()
.map(|fragment| fragment.metadata().clone())
.collect();
let operation = Operation::Update {
removed_fragment_ids: Vec::new(),
updated_fragments,
new_fragments: Vec::new(),
fields_modified,
compacted_sstables: Vec::new(),
fields_for_preserving_frag_bitmap: Vec::new(),
update_mode: Some(UpdateMode::RewriteColumns),
inserted_rows_filter: None,
updated_fragment_offsets: None,
};
let read_version = dataset.version().version;
CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset)))
.execute(Transaction::new(read_version, operation, None))
.await
.unwrap();
}
/// Refresh never computes a computed column: every row it writes, on a
/// rebuild, an append and a rewrite, carries NULL there, and the
/// declaration survives all three.
#[tokio::test]
async fn test_computed_columns_are_written_null_and_kept() {
let conn = connect("memory://").execute().await.unwrap();
let view = refreshed_computed_view(&conn).await;
assert_eq!(unfilled(&view).await, 3);
append_people(&conn, vec![4, 5], vec!["d", "e"]).await;
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(unfilled(&view).await, 5);
conn.open_table("people")
.execute()
.await
.unwrap()
.update()
.column("name", "'z'")
.only_if("id = 1")
.execute()
.await
.unwrap();
view.refresh().execute().await.unwrap();
assert_eq!(unfilled(&view).await, 5);
assert_eq!(read(view.table(), "id").await, vec![1, 2, 3, 4, 5]);
let schema = view.table().schema().await.unwrap();
assert!(
crate::table::computed_columns::function_bindings(&schema)
.unwrap()
.iter()
.any(|b| b.binding_id() == "fb_1"),
"the binding envelope was lost"
);
assert!(
computed_column_from_field(schema.field_with_name("emb").unwrap()).is_some(),
"the declaration was lost"
);
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
}
/// The fill job's commit rewrites only computed columns. It is the one
/// commit on a view that is not drift: the next refresh carries on from
/// its watermark instead of rebuilding, which would null what the fill
/// just wrote.
#[tokio::test]
async fn test_a_computed_column_fill_is_not_drift() {
let conn = connect("memory://").execute().await.unwrap();
let view = refreshed_computed_view(&conn).await;
commit_column_rewrite(&view, &["emb"]).await;
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
commit_column_rewrite(&view, &["emb"]).await;
append_people(&conn, vec![4], vec!["d"]).await;
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 1);
assert_eq!(read(view.table(), "id").await, vec![1, 2, 3, 4]);
}
/// A column rewrite that reaches a projected column is drift like any
/// other write: refresh certifies those columns and must recompute them.
#[tokio::test]
async fn test_a_rewrite_of_a_projected_column_is_drift() {
let conn = connect("memory://").execute().await.unwrap();
let view = refreshed_computed_view(&conn).await;
commit_column_rewrite(&view, &["emb", "name"]).await;
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::Rebuild
);
}
/// The declaration contract is checked before any refresh mutation: a
/// missing binding envelope and a column that lost its declaration both
/// fail closed.
#[tokio::test]
async fn test_a_broken_declaration_is_refused_before_refresh() {
let conn = connect("memory://").execute().await.unwrap();
let view = refreshed_computed_view(&conn).await;
let native = view.table().as_native().unwrap();
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
dataset
.update_schema_metadata(vec![(
crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(),
None,
)])
.await
.unwrap();
let err = view.refresh().execute().await.unwrap_err().to_string();
assert!(err.contains("references missing binding 'fb_1'"), "{err}");
let conn = connect("memory://").execute().await.unwrap();
let view = refreshed_computed_view(&conn).await;
let native = view.table().as_native().unwrap();
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
dataset
.replace_field_metadata(vec![(
dataset.schema().field("emb").unwrap().id as u32,
HashMap::new(),
)])
.await
.unwrap();
let err = view.refresh().execute().await.unwrap_err().to_string();
assert!(err.contains("does not match binding 'fb_1'"), "{err}");
}
/// An input the view does not project is materialized on every refresh
/// path, before the provenance column, with the source's values.
#[tokio::test]
async fn test_internal_inputs_are_materialized_and_refreshed() {
use crate::materialized_view::tests::{computed_field, strict_people, test_binding};
let conn = connect("memory://").execute().await.unwrap();
let source = strict_people(&conn).await;
let mut prepared = crate::materialized_view::prepare_declaration(
&source,
Some(&[("id".to_string(), "id".to_string())]),
None,
None,
)
.await
.unwrap();
let input = prepared.input_column("name").unwrap();
let view = prepared
.with_computed_columns(
vec![(1, computed_field("emb", "fb_1", &input))],
&[test_binding("fb_1", &input, "emb")],
)
.unwrap()
.create("v")
.await
.unwrap();
let names: Vec<String> = view
.table()
.schema()
.await
.unwrap()
.fields()
.iter()
.map(|f| f.name().clone())
.collect();
assert_eq!(names, ["id", "emb", "__input_name", SOURCE_ROW_ID_COLUMN]);
let unfilled_inputs = || async {
view.table()
.count_rows(Some("__input_name IS NULL".to_string()))
.await
.unwrap()
};
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::Rebuild
);
assert_eq!(view.table().count_rows(None).await.unwrap(), 3);
assert_eq!(unfilled_inputs().await, 0);
let more = arrow_array::RecordBatch::try_new(
source.schema().await.unwrap(),
vec![
Arc::new(Int32Array::from(vec![4])),
Arc::new(arrow_array::StringArray::from(vec!["d"])),
],
)
.unwrap();
source.add(more).execute().await.unwrap();
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::Incremental
);
assert_eq!(unfilled_inputs().await, 0);
assert_eq!(
view.table()
.count_rows(Some("__input_name = 'd'".to_string()))
.await
.unwrap(),
1
);
source
.update()
.column("name", "'z'")
.only_if("id = 1")
.execute()
.await
.unwrap();
view.refresh().execute().await.unwrap();
assert_eq!(
view.table()
.count_rows(Some("__input_name = 'z'".to_string()))
.await
.unwrap(),
1
);
assert_eq!(
unfilled(&view).await,
4,
"rewritten and new rows are unfilled"
);
}
/// A SQL declaration is filled by `refresh_column` on the view, which
/// commits a data replacement; the next refresh continues from its
/// watermark and keeps what the fill wrote, and only rows the view added
/// since come back unfilled.
#[tokio::test]
async fn test_a_sql_fill_is_not_drift() {
use crate::materialized_view::tests::{people, sql_field};
let conn = connect("memory://").execute().await.unwrap();
let source = people(&conn).await;
let view = crate::materialized_view::prepare_declaration(
&source,
Some(&[("id".to_string(), "id".to_string())]),
None,
None,
)
.await
.unwrap()
.with_computed_columns(
vec![(
1,
sql_field("next", arrow_schema::DataType::Int32, "id + 1", r#"["id"]"#),
)],
&[],
)
.unwrap()
.create("v")
.await
.unwrap();
let filled = || async {
view.table()
.count_rows(Some("next = id + 1".to_string()))
.await
.unwrap()
};
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::Rebuild
);
assert_eq!(
view.table()
.refresh_column("next")
.await
.unwrap()
.rows_filled,
3
);
assert_eq!(filled().await, 3);
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
assert_eq!(filled().await, 3);
append_people(&conn, vec![4], vec!["d"]).await;
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::Incremental
);
assert_eq!(filled().await, 3);
assert_eq!(
view.table()
.refresh_column("next")
.await
.unwrap()
.rows_filled,
1
);
assert_eq!(filled().await, 4);
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
}
/// A fill of a nested computed column writes its child fields; that is
/// still a fill, not drift.
#[tokio::test]
async fn test_a_nested_sql_fill_is_not_drift() {
use crate::materialized_view::tests::{people, sql_field};
let conn = connect("memory://").execute().await.unwrap();
let source = people(&conn).await;
let payload = sql_field(
"payload",
arrow_schema::DataType::Struct(
vec![arrow_schema::Field::new(
"value",
arrow_schema::DataType::Utf8,
true,
)]
.into(),
),
"named_struct('value', name)",
r#"["name"]"#,
);
let view = crate::materialized_view::prepare_declaration(
&source,
Some(&[("name".to_string(), "name".to_string())]),
None,
None,
)
.await
.unwrap()
.with_computed_columns(vec![(1, payload)], &[])
.unwrap()
.create("v")
.await
.unwrap();
view.refresh().execute().await.unwrap();
assert_eq!(
view.table()
.refresh_column("payload")
.await
.unwrap()
.rows_filled,
3
);
assert_eq!(
view.refresh().execute().await.unwrap().mode,
RefreshMode::NoOp
);
assert_eq!(
view.table()
.count_rows(Some("payload.value = name".to_string()))
.await
.unwrap(),
3
);
}
}
+90 -128
View File
@@ -20,13 +20,13 @@ use lance_namespace::models::{
use crate::Error;
use crate::database::{
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions, JobInfo,
OpenTableRequest, ReadConsistency, TableNamesRequest,
CloneTableRequest, CreateTableMode, CreateTableRequest, Database, DatabaseOptions,
JobDescription, JobInfo, OpenTableRequest, ReadConsistency, TableNamesRequest,
};
use crate::error::Result;
use crate::function::{FunctionRegistrationRequest, FunctionVersion};
use crate::job::Job;
use crate::remote::job::{RemoteJob, job_state_to_client};
use crate::remote::job::{DescribeJobResponse, RemoteJob, job_state_to_client};
use crate::remote::util::stream_as_body;
use crate::table::BaseTable;
@@ -671,18 +671,11 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
Ok(response.dropped)
}
async fn open_job(&self, job_id: &str) -> Result<Job> {
let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string());
match crate::job::JobHandle::describe(&handle).await {
Ok(description) => Ok(Job::opened(Box::new(handle), description)),
Err(Error::Http {
status_code: Some(StatusCode::NOT_FOUND),
..
}) => Err(Error::JobNotFound {
job_id: job_id.to_string(),
}),
Err(err) => Err(err),
}
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
self.client.clone(),
job_id.to_string(),
))))
}
async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
@@ -719,6 +712,31 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
Ok(out)
}
async fn get_job(&self, job_id: &str) -> Result<Option<JobDescription>> {
let req = self
.client
.post("/v1/jobs/describe")
.json(&serde_json::json!({ "job_id": job_id }));
let (request_id, rsp) = self.client.send(req).await?;
let rsp = match self.client.check_response(&request_id, rsp).await {
Ok(rsp) => rsp,
Err(Error::Http {
status_code: Some(StatusCode::NOT_FOUND),
..
}) => return Ok(None),
Err(err) => return Err(err),
};
let body: DescribeJobResponse = rsp.json().await.err_to_http(request_id)?;
Ok(Some(JobDescription {
job_id: body.job_id,
job_type: body.job_type,
state: job_state_to_client(&body.job_state),
creation_ms: body.creation_ms,
spec: body.spec,
failure: body.failure.map(|reported| reported.into_job_failure()),
}))
}
async fn cancel_job(&self, job_id: &str) -> Result<bool> {
let req = self
.client
@@ -735,6 +753,21 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
}
async fn job_history(&self, job_id: Option<&str>) -> Result<Vec<arrow_array::RecordBatch>> {
let mut body = serde_json::json!({});
if let Some(job_id) = job_id {
body["job_id"] = serde_json::Value::String(job_id.to_string());
}
let req = self.client.post("/v1/jobs/query_events").json(&body);
let (request_id, rsp) = self.client.send(req).await?;
let rsp = self.client.check_response(&request_id, rsp).await?;
let bytes = rsp.bytes().await.err_to_http(request_id)?;
let reader = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
reader
.collect::<std::result::Result<Vec<_>, _>>()
.map_err(Into::into)
}
async fn execute_query_async(
&self,
query: &str,
@@ -1274,7 +1307,6 @@ mod tests {
use crate::{
Connection, Error,
database::CreateTableMode,
job::JobEventsRequest,
remote::{ARROW_STREAM_CONTENT_TYPE, ClientConfig, HeaderProvider, JSON_CONTENT_TYPE},
};
@@ -2623,7 +2655,7 @@ mod tests {
}
#[tokio::test]
async fn test_open_job() {
async fn test_get_job() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/jobs/describe");
@@ -2637,55 +2669,51 @@ mod tests {
)
.unwrap()
});
// Opening populates the handle, so the accessors answer without a
// second round trip.
let job = conn.open_job("job-1").await.unwrap();
assert_eq!(job.id(), Some("job-1"));
assert_eq!(job.job_type().as_deref(), Some("create_index"));
assert_eq!(job.state().as_deref(), Some("failed"));
assert_eq!(job.creation_ms(), Some(1000));
assert_eq!(job.spec().unwrap()["column"], "vec");
assert!(job.result().is_none());
let failure = job.failure().unwrap();
let job = conn.get_job("job-1").await.unwrap().unwrap();
assert_eq!(job.job_id, "job-1");
assert_eq!(job.job_type, "create_index");
assert_eq!(job.state, "failed");
assert_eq!(job.creation_ms, 1000);
assert_eq!(job.spec["column"], "vec");
let failure = job.failure.unwrap();
assert_eq!(failure.phase.as_deref(), Some("execute"));
assert_eq!(failure.message.as_deref(), Some("worker died"));
assert_eq!(failure.retryable, Some(true));
}
#[tokio::test]
async fn test_open_job_reports_the_terminal_result() {
let conn = Connection::new_with_handler(|_| {
http::Response::builder()
.status(200)
.body(
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "DONE", "creation_ms": 1000, "result": {"rows_assigned": 1000000, "rows_failed": 0}}"#,
)
.unwrap()
});
let job = conn.open_job("job-1").await.unwrap();
assert_eq!(job.state().as_deref(), Some("finished"));
let result = job.result().unwrap();
assert_eq!(result["rows_assigned"], 1_000_000);
assert_eq!(result["rows_failed"], 0);
}
#[tokio::test]
async fn test_open_job_missing_fails() {
async fn test_get_job_missing_is_none() {
let conn = Connection::new_with_handler(|_| {
http::Response::builder()
.status(404)
.body("no such job")
.unwrap()
});
let err = conn.open_job("nope").await.unwrap_err();
assert!(
matches!(&err, Error::JobNotFound { job_id } if job_id == "nope"),
"{err:?}"
);
assert!(conn.get_job("nope").await.unwrap().is_none());
}
#[tokio::test]
async fn test_job_events_scope_to_that_job() {
async fn test_cancel_job() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.url().path(), "/v1/jobs/cancel");
http::Response::builder()
.status(200)
.body(r#"{"job_id": "job-1"}"#)
.unwrap()
});
assert!(conn.cancel_job("job-1").await.unwrap());
let conn = Connection::new_with_handler(|_| {
http::Response::builder()
.status(404)
.body("no such job")
.unwrap()
});
assert!(!conn.cancel_job("nope").await.unwrap());
}
#[tokio::test]
async fn test_job_history_parses_arrow_stream() {
let schema = Arc::new(Schema::new(vec![Field::new(
"state",
DataType::Utf8,
@@ -2694,91 +2722,29 @@ mod tests {
let batch = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(arrow_array::StringArray::from(vec![
"claim_complete",
"created", "done",
]))],
)
.unwrap();
let mut events = Vec::new();
let mut body = Vec::new();
{
let mut writer =
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut body, &schema).unwrap();
writer.write(&batch).unwrap();
writer.finish().unwrap();
}
let conn = Connection::new_with_handler(move |request| {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
if request.url().path() == "/v1/jobs/describe" {
return http::Response::builder()
.status(200)
.body(
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 1}"#
.as_bytes()
.to_vec(),
)
.unwrap();
}
assert_eq!(request.url().path(), "/v1/jobs/query_events");
// The handle supplies job_id; the caller only narrows the query.
assert_eq!(body["job_id"], "job-1");
assert_eq!(body["limit"], 500);
assert_eq!(body["filter"], "state = 'claim_complete'");
http::Response::builder()
.status(200)
.body(events.clone())
.unwrap()
});
let job = conn.open_job("job-1").await.unwrap();
let batches = job
.events(
JobEventsRequest::default()
.limit(500)
.filter("state = 'claim_complete'"),
)
.await
.unwrap();
assert_eq!(batches.len(), 1);
assert_eq!(batches[0].num_rows(), 1);
}
#[tokio::test]
async fn test_job_events_keep_the_schema_when_nothing_matches() {
let schema = Arc::new(Schema::new(vec![Field::new(
"state",
DataType::Utf8,
false,
)]));
let mut events = Vec::new();
{
let mut writer =
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
writer.finish().unwrap();
}
let conn = Connection::new_with_handler(move |request| {
if request.url().path() == "/v1/jobs/describe" {
return http::Response::builder()
.status(200)
.body(
r#"{"job_id": "job-1", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 1}"#
.as_bytes()
.to_vec(),
)
.unwrap();
}
let body: serde_json::Value =
let req_body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
// Only the job id when the caller narrows nothing.
assert_eq!(body, serde_json::json!({ "job_id": "job-1" }));
assert_eq!(req_body["job_id"], "job-1");
http::Response::builder()
.status(200)
.body(events.clone())
.body(body.clone())
.unwrap()
});
let job = conn.open_job("job-1").await.unwrap();
let batches = job.events(JobEventsRequest::default()).await.unwrap();
let batches = conn.job_history(Some("job-1")).await.unwrap();
assert_eq!(batches.len(), 1);
assert_eq!(batches[0].num_rows(), 0);
assert_eq!(batches[0].schema(), schema);
assert_eq!(batches[0].num_rows(), 2);
}
#[tokio::test]
@@ -2973,9 +2939,7 @@ mod tests {
let polls_ref = polls.clone();
let conn = Connection::new_with_handler(move |request| {
assert_eq!(request.url().path(), "/v1/jobs/describe");
// Two in-progress answers: one for the load, one for the first
// status poll.
let state = if polls_ref.fetch_add(1, Ordering::SeqCst) < 2 {
let state = if polls_ref.fetch_add(1, Ordering::SeqCst) == 0 {
"IN_PROGRESS"
} else {
"DONE"
@@ -2988,13 +2952,11 @@ mod tests {
))
.unwrap()
});
let job = conn.open_job("job-1").await.unwrap();
let job = conn.job("job-1").unwrap();
assert_eq!(job.id(), Some("job-1"));
// Opening already answered the state; no extra call needed for it.
assert_eq!(job.state().as_deref(), Some("running"));
assert_eq!(job.status().await.unwrap(), "running");
job.wait().await.unwrap();
assert_eq!(job.status().await.unwrap(), "finished");
assert!(polls.load(Ordering::SeqCst) >= 4);
assert!(polls.load(Ordering::SeqCst) >= 3);
}
}
+5 -55
View File
@@ -5,15 +5,13 @@
use std::time::Duration;
use arrow_array::RecordBatch;
use async_trait::async_trait;
use tokio::time::sleep;
use serde::Deserialize;
use crate::database::JobDescription;
use crate::error::{Error, JobFailure, Result};
use crate::job::{JobEventsRequest, JobHandle, TerminalResult};
use crate::job::{JobHandle, TerminalResult};
use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
/// Delay before the second job-state poll; doubles up to [`MAX_POLL_INTERVAL`].
@@ -88,7 +86,7 @@ pub(super) struct DescribeJobResponse {
#[serde(default)]
pub(super) spec: serde_json::Value,
#[serde(default)]
pub(super) result: Option<serde_json::Value>,
result: Option<serde_json::Value>,
#[serde(default)]
pub(super) failure: Option<ReportedFailure>,
}
@@ -112,39 +110,6 @@ impl DescribeJobResponse {
fn into_terminal_result(self, request_id: String) -> TerminalResult {
TerminalResult::remote(self.result, request_id)
}
/// The public description this wire envelope stands for.
pub(super) fn into_description(self) -> JobDescription {
JobDescription {
job_id: self.job_id,
job_type: self.job_type,
state: JobState::from(self.job_state.as_str()).client_label(),
creation_ms: self.creation_ms,
spec: self.spec,
result: self.result,
failure: self.failure.map(ReportedFailure::into_job_failure),
}
}
}
/// One `/v1/jobs/query_events` round trip.
pub(super) async fn fetch_job_events<S: HttpSend>(
client: &RestfulLanceDbClient<S>,
body: serde_json::Value,
) -> Result<Vec<RecordBatch>> {
let request = client.post("/v1/jobs/query_events").json(&body);
let (request_id, response) = client.send(request).await?;
let response = client.check_response(&request_id, response).await?;
let bytes = response.bytes().await.err_to_http(request_id)?;
let reader = arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(bytes), None)?;
let schema = reader.schema();
let mut batches = reader.collect::<std::result::Result<Vec<_>, _>>()?;
// A query that matched nothing still describes the event columns.
// Keep that schema so callers can build a typed empty result.
if batches.is_empty() {
batches.push(RecordBatch::new_empty(schema));
}
Ok(batches)
}
pub struct RemoteJob<S: HttpSend> {
@@ -158,7 +123,7 @@ impl<S: HttpSend> RemoteJob<S> {
}
/// One `/v1/jobs/describe` round trip.
async fn fetch_description(&self) -> Result<(String, DescribeJobResponse)> {
async fn describe(&self) -> Result<(String, DescribeJobResponse)> {
let request = self
.client
.post("/v1/jobs/describe")
@@ -183,28 +148,13 @@ impl<S: HttpSend> JobHandle for RemoteJob<S> {
}
async fn status(&self) -> Result<String> {
Ok(self.fetch_description().await?.1.state().client_label())
}
async fn describe(&self) -> Result<JobDescription> {
Ok(self.fetch_description().await?.1.into_description())
}
async fn events(&self, request: JobEventsRequest) -> Result<Vec<RecordBatch>> {
let mut body = serde_json::json!({ "job_id": self.job_id });
if let Some(limit) = request.limit {
body["limit"] = serde_json::Value::from(limit);
}
if let Some(filter) = request.filter {
body["filter"] = serde_json::Value::String(filter);
}
fetch_job_events(&self.client, body).await
Ok(self.describe().await?.1.state().client_label())
}
async fn wait(&self) -> Result<TerminalResult> {
let mut interval = INITIAL_POLL_INTERVAL;
loop {
let (request_id, description) = self.fetch_description().await?;
let (request_id, description) = self.describe().await?;
match description.state() {
JobState::Done => return Ok(description.into_terminal_result(request_id)),
JobState::Failed => {
-77
View File
@@ -264,17 +264,6 @@ impl<S: HttpSend> crate::job::JobHandle for FreshnessJob<S> {
crate::job::JobHandle::status(&self.inner).await
}
async fn describe(&self) -> Result<crate::database::JobDescription> {
crate::job::JobHandle::describe(&self.inner).await
}
async fn events(
&self,
request: crate::job::JobEventsRequest,
) -> Result<Vec<arrow_array::RecordBatch>> {
crate::job::JobHandle::events(&self.inner, request).await
}
async fn wait(&self) -> Result<crate::job::TerminalResult> {
let result = crate::job::JobHandle::wait(&self.inner).await?;
let version = self.version.read().await;
@@ -7993,72 +7982,6 @@ mod tests {
);
}
/// The refresh handle is wrapped for read-freshness tracking, so it has to
/// forward the detail APIs too -- this is the job an operator is holding
/// when a backfill goes quiet.
#[tokio::test]
async fn test_refresh_job_handle_reports_detail_and_events() {
let schema = Arc::new(Schema::new(vec![Field::new(
"state",
DataType::Utf8,
false,
)]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![Arc::new(arrow_array::StringArray::from(vec![
"claim_complete",
]))],
)
.unwrap();
let mut events = Vec::new();
{
let mut writer =
arrow_ipc::writer::StreamWriter::try_new(&mut events, &schema).unwrap();
writer.write(&batch).unwrap();
writer.finish().unwrap();
}
let table = Table::new_with_handler("my_table", move |request| {
match request.url().path() {
"/v1/table/my_table/backfill_column" => http::Response::builder()
.status(202)
.body(br#"{"job_id": "j-42"}"#.to_vec())
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(
r#"{"job_id": "j-42", "job_type": "refresh_column", "job_state": "IN_PROGRESS", "creation_ms": 7, "spec": {"column": "doubled"}}"#
.as_bytes()
.to_vec(),
)
.unwrap(),
"/v1/jobs/query_events" => {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap())
.unwrap();
assert_eq!(body["job_id"], "j-42");
http::Response::builder()
.status(200)
.body(events.clone())
.unwrap()
}
other => panic!("unexpected path {other}"),
}
});
let job = table.refresh_column_async("doubled").await.unwrap();
job.refresh().await.unwrap();
assert_eq!(job.state().as_deref(), Some("running"));
assert_eq!(job.job_type().as_deref(), Some("refresh_column"));
assert_eq!(job.creation_ms(), Some(7));
assert_eq!(job.spec().unwrap()["column"], "doubled");
let batches = job
.events(crate::job::JobEventsRequest::default())
.await
.unwrap();
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 1);
}
#[tokio::test]
async fn test_refresh_submission_uses_add_columns_version_fence() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
+2 -3
View File
@@ -2804,7 +2804,7 @@ impl NativeTable {
namespace_client: Option<Arc<dyn LanceNamespace>>,
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
) -> Result<Self> {
let batches = computed_columns::admit_create_source(batches)?;
computed_columns::ensure_no_foreign_declarations(batches.arrow_schema().fields())?;
// Default params uses format v1.
let params = params.unwrap_or(WriteParams {
..Default::default()
@@ -2904,7 +2904,6 @@ impl NativeTable {
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
session: Option<Arc<lance::session::Session>>,
) -> Result<Self> {
let batches = computed_columns::admit_create_source(batches)?;
// Build table_id from namespace + name for the storage options provider
let mut table_id = namespace.clone();
table_id.push(name.to_string());
@@ -5678,7 +5677,7 @@ mod tests {
TableStatistics {
num_rows: 250,
num_indices: 0,
total_bytes: 8969,
total_bytes: 8925,
fragment_stats: FragmentStatistics {
num_fragments: 11,
num_small_fragments: 11,
+45 -322
View File
@@ -21,7 +21,6 @@
//! [`computed_columns`] and [`computed_column_from_field`] read declarations
//! back off a schema.
use futures::StreamExt;
use std::collections::{BTreeSet, HashMap, HashSet};
use std::sync::Arc;
@@ -760,33 +759,14 @@ fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonAr
Ok(field)
}
/// Whether two fields describe the same Function output.
///
/// `compare_identity` covers the field's own name and nullability. Struct
/// children carry both as part of the declaration and compare with it on. List
/// children do not: Lance rewrites a list item's name and nullability when it
/// writes, so a stored `fixed_size_list<item: float not null>` comes back as
/// `fixed_size_list<item: float>` and never matches the declaration again.
/// Comparing those by type alone keeps this agreeing with the server, which
/// draws the same distinction and is what accepted the column when it was
/// declared.
fn function_output_field_matches(
expected: &ArrowField,
actual: &ArrowField,
compare_identity: bool,
) -> bool {
if compare_identity
&& (expected.name() != actual.name() || expected.is_nullable() != actual.is_nullable())
{
return false;
}
match (expected.is_blob_v2(), actual.is_blob_v2()) {
(false, false) => function_output_type_matches(expected.data_type(), actual.data_type()),
(true, true) => {
fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
expected.name() == actual.name()
&& expected.is_nullable() == actual.is_nullable()
&& if expected.is_blob_v2() {
has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
} else {
function_output_type_matches(expected.data_type(), actual.data_type())
}
_ => false,
}
}
fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
@@ -799,19 +779,33 @@ fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool
&& expected
.iter()
.zip(actual)
.all(|(expected, actual)| function_output_field_matches(expected, actual, true))
.all(|(expected, actual)| function_output_field_matches(expected, actual))
}
(DataType::List(expected), DataType::List(actual))
| (DataType::LargeList(expected), DataType::LargeList(actual)) => {
function_output_field_matches(expected, actual, false)
function_output_field_matches(expected, actual)
}
(
DataType::FixedSizeList(expected, expected_size),
DataType::FixedSizeList(actual, actual_size),
) => expected_size == actual_size && function_output_field_matches(expected, actual, false),
) => expected_size == actual_size && function_output_field_matches(expected, actual),
(DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
expected_sorted == actual_sorted
&& function_output_field_matches(expected, actual, true)
expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
}
_ => false,
}
}
fn function_output_type_has_blob(data_type: &DataType) -> bool {
match data_type {
DataType::Struct(fields) => fields
.iter()
.any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
DataType::List(field)
| DataType::LargeList(field)
| DataType::FixedSizeList(field, _)
| DataType::Map(field, _) => {
field.is_blob_v2() || function_output_type_has_blob(field.data_type())
}
_ => false,
}
@@ -897,13 +891,16 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
binding.binding_id()
)));
}
let type_matches = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
has_supported_blob_v2_layout(field)
let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
(has_supported_blob_v2_layout(field), true)
} else {
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
function_output_type_matches(&expected_type, field.data_type())
(
function_output_type_matches(&expected_type, field.data_type()),
function_output_type_has_blob(&expected_type),
)
};
if !type_matches {
return Err(invalid_function(format!(
@@ -934,16 +931,19 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
binding.binding_id()
)));
}
// Rebuild from the declaration rather than from the stored field. The
// stored field carries Lance's write-time normalization, which would
// never round-trip back to the schema the binding recorded -- the same
// reason list children compare by type above. Whether the column on
// disk still matches is settled by that comparison, not here.
output_fields.push(function_output_field(
field.name(),
true,
&output.arrow_type,
)?);
if has_semantic_blob {
output_fields.push(function_output_field(
field.name(),
true,
&output.arrow_type,
)?);
} else {
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
ArrowField::new(field.name().clone(), field.data_type().clone(), true),
]))
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
output_fields.push(json.fields.into_iter().next().unwrap());
}
}
if let Some(assignment) = binding.assignment() {
if binding
@@ -1339,106 +1339,6 @@ pub(crate) fn ensure_batch_writes_no_computed_values(
Ok(())
}
/// Validate every computed-column declaration `schema` carries against the
/// schema itself: every field with declaration metadata is a complete
/// declaration, a SQL declaration re-plans to the field it declares, a
/// Function declaration satisfies the binding contract, and no declaration
/// reads another computed column. What passes here is what `refresh_column`
/// can execute.
pub(crate) fn ensure_declarations_are_planned(schema: &ArrowSchema) -> Result<()> {
let invalid = |message: String| Error::InvalidInput { message };
// A field with any declaration key is a declaration; a partial one is
// not "no declaration", it is a broken one.
for field in schema.fields() {
if field.metadata().keys().any(|k| is_declaration_key(k))
&& computed_column_from_field(field).is_none()
{
return Err(invalid(format!(
"field '{}' carries an incomplete computed-column declaration",
field.name()
)));
}
}
let declared: HashSet<String> = computed_columns(schema)
.into_iter()
.map(|c| c.name)
.collect();
for column in computed_columns(schema) {
let field = schema.field_with_name(&column.name)?;
if !field.is_nullable() {
return Err(invalid(format!(
"computed column '{}' must be nullable until a refresh fills it",
column.name
)));
}
match &column.kind {
ComputedColumnKind::Sql { expression } => {
let others: Vec<ArrowField> = schema
.fields()
.iter()
.filter(|f| f.name() != &column.name)
.map(|f| f.as_ref().clone())
.collect();
let bound = bind(Arc::new(ArrowSchema::new(others)), &column.name, expression)?;
if let Some(input) = bound.roots.iter().find(|r| declared.contains(*r)) {
return Err(invalid(format!(
"computed column '{}' reads computed column '{input}'",
column.name
)));
}
if &bound.data_type != field.data_type() {
return Err(invalid(format!(
"computed column '{}' is declared as {} but its expression yields {}",
column.name,
field.data_type(),
bound.data_type
)));
}
let mut declared_inputs = column.inputs.clone();
declared_inputs.sort();
if declared_inputs != bound.inputs {
return Err(invalid(format!(
"computed column '{}' declares inputs {:?} but its expression reads {:?}",
column.name, declared_inputs, bound.inputs
)));
}
}
ComputedColumnKind::Function { binding_id, .. } => {
// The binding validator resolves each input's leaf; the
// no-computed-input rule is about the root it hangs from.
let bindings = function_bindings(schema)?;
let Some(binding) = bindings.iter().find(|b| b.binding_id() == binding_id) else {
continue; // reported by the binding validator below
};
// Roots come from the canonical path parser: a quoted
// top-level name may itself contain a dot.
if let Some(input) = binding
.inputs()
.iter()
.filter_map(|input| resolve_field_path(schema, &input.field_path).ok())
.map(|resolved| resolved.root.name().as_str())
.find(|r| declared.contains(*r))
{
return Err(invalid(format!(
"computed column '{}' reads computed column '{input}'",
column.name
)));
}
}
ComputedColumnKind::Unrecognized { kind } => {
return Err(Error::NotSupported {
message: format!(
"computed column '{}' is defined by '{kind}', which this version \
of lancedb cannot fill",
column.name
),
});
}
}
}
ensure_supported_function_metadata(schema)
}
/// Reject fields carrying declaration metadata that did not come through
/// [`plan`]. One authority for creation, overwrite and raw transforms.
pub(crate) fn ensure_no_foreign_declarations<'a>(
@@ -1897,54 +1797,6 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st
.unwrap();
}
/// Admit a table's initial data: every declaration it carries is validated,
/// and the stream refuses any batch with values in a computed column, whose
/// values come from refresh alone. One boundary for every way a table is
/// created.
pub(crate) fn admit_create_source<S: lance_datafusion::utils::StreamingWriteSource>(
batches: S,
) -> Result<UnfilledDeclarations<S>> {
let schema = batches.arrow_schema();
ensure_declarations_are_planned(&schema)?;
let declared = computed_columns(&schema)
.into_iter()
.map(|c| c.name)
.collect();
Ok(UnfilledDeclarations {
inner: batches,
declared,
})
}
/// A write source whose computed columns must arrive unfilled.
pub(crate) struct UnfilledDeclarations<S> {
inner: S,
declared: Vec<String>,
}
impl<S: lance_datafusion::utils::StreamingWriteSource> lance_datafusion::utils::StreamingWriteSource
for UnfilledDeclarations<S>
{
fn arrow_schema(&self) -> SchemaRef {
self.inner.arrow_schema()
}
fn into_stream(self) -> datafusion_physical_plan::SendableRecordBatchStream {
if self.declared.is_empty() {
return self.inner.into_stream();
}
let schema = self.inner.arrow_schema();
let declared = self.declared;
let stream = self.inner.into_stream().map(move |batch| {
let batch = batch?;
ensure_batch_writes_no_computed_values(&declared, &batch)
.map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?;
Ok(batch)
});
Box::pin(datafusion_physical_plan::stream::RecordBatchStreamAdapter::new(schema, stream))
}
}
#[cfg(test)]
mod tests {
/// The gate's reproducer: the validator applies the same schema-level
@@ -1963,69 +1815,6 @@ mod tests {
assert!(super::validate_declarations(schema, &declarations).is_err());
}
#[test]
fn list_children_match_by_type_but_struct_children_by_identity() {
use arrow_schema::Field as F;
// Lance rewrites a list item's name and nullability on write, so the
// stored field is no longer identical to what was declared. Comparing
// those by type keeps a table with a vector output usable.
let declared =
DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, false)), 4);
let stored = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 4);
assert!(super::function_output_type_matches(&declared, &stored));
let renamed =
DataType::FixedSizeList(Arc::new(F::new("element", DataType::Float32, true)), 4);
assert!(super::function_output_type_matches(&declared, &renamed));
// The dimension is still part of the declaration.
let resized = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 8);
assert!(!super::function_output_type_matches(&declared, &resized));
// Struct children keep comparing by name and nullability.
let struct_declared =
DataType::Struct(vec![F::new("changed", DataType::Boolean, false)].into());
let struct_nullable =
DataType::Struct(vec![F::new("changed", DataType::Boolean, true)].into());
let struct_renamed =
DataType::Struct(vec![F::new("altered", DataType::Boolean, false)].into());
assert!(super::function_output_type_matches(
&struct_declared,
&struct_declared
));
assert!(!super::function_output_type_matches(
&struct_declared,
&struct_nullable
));
assert!(!super::function_output_type_matches(
&struct_declared,
&struct_renamed
));
// A list nested inside a struct gets the list rule.
let nested_declared = DataType::Struct(
vec![F::new(
"tokens",
DataType::List(Arc::new(F::new("item", DataType::Utf8, false))),
true,
)]
.into(),
);
let nested_stored = DataType::Struct(
vec![F::new(
"tokens",
DataType::List(Arc::new(F::new("item", DataType::Utf8, true))),
true,
)]
.into(),
);
assert!(super::function_output_type_matches(
&nested_declared,
&nested_stored
));
}
#[test]
fn output_arrow_type_grammar_matches_the_shared_golden() {
let golden: serde_json::Value = serde_json::from_str(include_str!(
@@ -2795,8 +2584,6 @@ mod tests {
);
}
/// A create carries a declaration only if it re-plans completely; this
/// one lacks its inputs and is refused before its forged value matters.
#[tokio::test]
async fn test_create_table_cannot_inject_a_declaration() {
let conn = connect("memory://").execute().await.unwrap();
@@ -2824,7 +2611,7 @@ mod tests {
.await
.unwrap_err();
assert!(
matches!(&err, Error::InvalidInput { message } if message.contains("computed column 'doubled'")),
matches!(&err, Error::InvalidInput { message } if message.contains("computed()")),
"{err:?}"
);
}
@@ -3502,70 +3289,6 @@ mod tests {
assert!(output_schema.field(0).is_blob_v2());
}
#[test]
fn binding_accepts_a_lance_normalized_list_child() {
// The whole guard, not just the type helper: this also reaches the
// output-schema comparison at the end of ensure_binding_matches_schema,
// which used to rebuild the schema from the stored field and so failed
// on exactly the same normalization.
let input = ArrowField::new("value", DataType::Int64, false);
let application = FunctionApplication::from_json(
&serde_json::json!({
"function": {"name": "embed", "version": "fv_embed"},
"inputs": [{
"parameter": "value",
"kind": "column",
"value": {"path": "value"}
}],
"output": {
"kind": "scalar",
"arrow_type": "fixed_size_list<float32, 4>",
"nullable": false
}
})
.to_string(),
)
.unwrap();
let plan = plan_function_application(
&ArrowSchema::new(vec![input.clone()]),
&application,
Some("embedding"),
)
.unwrap();
let binding = binding_from_plan(&plan);
// The declaration says the item is non-nullable; Lance rewrites it to
// nullable on write, so this is what the column looks like on disk.
let stored = DataType::FixedSizeList(
Arc::new(ArrowField::new("item", DataType::Float32, true)),
4,
);
let output = ArrowField::new("embedding", stored, true).with_metadata(
function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]),
);
ensure_binding_matches_schema(&ArrowSchema::new(vec![input.clone(), output]), &binding)
.unwrap();
// A different element type is still a mismatch.
let wrong = ArrowField::new(
"embedding",
DataType::FixedSizeList(
Arc::new(ArrowField::new("item", DataType::Float64, true)),
4,
),
true,
)
.with_metadata(function_computed_column_metadata(
binding.binding_id(),
0,
&["value".into()],
));
assert!(
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, wrong]), &binding).is_err()
);
}
#[test]
fn test_blob_scalar_binding_accepts_full_logical_layout() {
let input = crate::blob("image", false);
@@ -5,17 +5,12 @@
//!
//! [`super::cast::cast_to_table_schema`] calls [`coerce_blob_expr`].
use std::fmt;
use std::hash::{Hash, Hasher};
use std::sync::Arc;
use arrow_array::{Array, BooleanArray, RecordBatch};
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
use arrow_select::nullif::nullif;
use arrow_schema::{DataType, Field, FieldRef, Fields};
use datafusion::functions::core::{get_field, named_struct};
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
use datafusion_expr::ColumnarValue;
use datafusion_physical_expr::ScalarFunctionExpr;
use datafusion_physical_expr::expressions::{CastExpr, Literal};
use datafusion_physical_plan::PhysicalExpr;
@@ -138,102 +133,16 @@ pub(super) fn coerce_blob_expr(
ns_args.push(value);
}
let built: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
let expr: Arc<dyn PhysicalExpr> = Arc::new(ScalarFunctionExpr::new(
&format!("named_struct({})", table_field.name()),
named_struct(),
ns_args,
table_field.clone(),
config.clone(),
));
// `named_struct` always yields a valid struct, so a null input would land
// as a row that set neither `data` nor `uri` -- not an absent blob but a
// malformed one, which Lance rejects on write.
let expr: Arc<dyn PhysicalExpr> = Arc::new(AbsentBlobIsNull {
source: input_expr,
built,
field: table_field.clone(),
});
Ok((expr, table_field.clone()))
}
/// Carries the source column's nullity onto the struct built for it.
///
/// This is its own expression rather than a `CASE` because the projection
/// takes its output field from `return_field`, and the generic implementation
/// rebuilds a bare field -- which would drop the `lance.blob.v2` extension
/// metadata and stop the column being recognised as a blob at all.
#[derive(Debug, Clone)]
struct AbsentBlobIsNull {
source: Arc<dyn PhysicalExpr>,
built: Arc<dyn PhysicalExpr>,
field: FieldRef,
}
impl fmt::Display for AbsentBlobIsNull {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "absent_blob_is_null({}, {})", self.source, self.built)
}
}
impl PartialEq for AbsentBlobIsNull {
fn eq(&self, other: &Self) -> bool {
self.source.eq(&other.source) && self.built.eq(&other.built) && self.field == other.field
}
}
impl Eq for AbsentBlobIsNull {}
impl Hash for AbsentBlobIsNull {
fn hash<H: Hasher>(&self, state: &mut H) {
self.source.hash(state);
self.built.hash(state);
self.field.hash(state);
}
}
impl PhysicalExpr for AbsentBlobIsNull {
fn return_field(&self, _input_schema: &Schema) -> datafusion_common::Result<FieldRef> {
Ok(self.field.clone())
}
fn nullable(&self, _input_schema: &Schema) -> datafusion_common::Result<bool> {
Ok(true)
}
fn evaluate(&self, batch: &RecordBatch) -> datafusion_common::Result<ColumnarValue> {
let rows = batch.num_rows();
let built = self.built.evaluate(batch)?.into_array(rows)?;
let source = self.source.evaluate(batch)?.into_array(rows)?;
let Some(nulls) = source.logical_nulls() else {
return Ok(ColumnarValue::Array(built));
};
// `nullif` nulls the rows the mask marks true, which is where the
// source had no value.
let absent = BooleanArray::new(!nulls.inner(), None);
Ok(ColumnarValue::Array(nullif(built.as_ref(), &absent)?))
}
fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
vec![&self.source, &self.built]
}
fn with_new_children(
self: Arc<Self>,
children: Vec<Arc<dyn PhysicalExpr>>,
) -> datafusion_common::Result<Arc<dyn PhysicalExpr>> {
Ok(Arc::new(Self {
source: children[0].clone(),
built: children[1].clone(),
field: self.field.clone(),
}))
}
fn fmt_sql(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self}")
}
}
enum BlobInputShape<'a> {
Bytes,
String,
@@ -404,11 +313,6 @@ mod tests {
let data = image.column_by_name("data").unwrap();
assert!(!data.is_null(0));
assert!(data.is_null(1));
// The row itself has to be null, not merely a struct whose children
// are. A present-but-empty struct set neither `data` nor `uri`, which
// Lance rejects as malformed rather than reading as an absent blob.
assert!(!image.is_null(0));
assert!(image.is_null(1));
}
#[tokio::test]
+2 -10
View File
@@ -19,7 +19,6 @@ use lancedb::{
connect, connect_namespace,
database::listing::{
ListingDatabaseOptions, NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS,
OPT_NEW_TABLE_STORAGE_VERSION,
},
query::{ExecutableQuery, QueryBase},
table::{AddDataMode, CompactionOptions, OptimizeAction, OptimizeStats, WriteOptions},
@@ -147,10 +146,7 @@ async fn non_blob_table_keeps_default_format_and_row_id_setting() -> Result<()>
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
let table = db.create_empty_table("t", schema).execute().await?;
assert_eq!(
storage_format_version(&table).await,
LanceFileVersion::Stable.resolve()
);
assert!(!supports_blob_v2(storage_format_version(&table).await));
assert!(!uses_stable_row_ids(&table).await);
Ok(())
}
@@ -813,11 +809,7 @@ async fn fetch_blobs_rejects_unknown_column() -> Result<()> {
#[tokio::test]
async fn fetch_blobs_rejects_legacy_v1_blob_column() -> Result<()> {
let tmp = tempdir().unwrap();
// Legacy v1 blob columns are only writable at file version <= 2.1.
let db = connect(tmp.path().to_str().unwrap())
.storage_options([(OPT_NEW_TABLE_STORAGE_VERSION, "2.1")])
.execute()
.await?;
let db = connect(tmp.path().to_str().unwrap()).execute().await?;
let legacy = Field::new("image", DataType::LargeBinary, true).with_metadata(
std::collections::HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]),
);