From 54efd4949c344d6f8fcbda09019f6c5435d18959 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sun, 20 Sep 2026 01:27:13 -0700 Subject: [PATCH 01/15] chore: update lance dependency to v13.0.0-beta.7 (#4234) Updates the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core from v13.0.0-beta.6 to [v13.0.0-beta.7](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.7); no compatibility fixes are required. Validation: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check` passed. --- Cargo.lock | 88 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 ++++++++--------- java/pom.xml | 2 +- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 681bfd552..f7034c9dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3523,8 +3523,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5073,8 +5073,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arc-swap", "arrow", @@ -5146,8 +5146,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5169,7 +5169,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5183,7 +5183,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-schema", @@ -5192,8 +5192,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrayref", "crunchy", @@ -5203,8 +5203,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5241,8 +5241,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow", "arrow-array", @@ -5273,8 +5273,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow", "arrow-array", @@ -5291,8 +5291,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "proc-macro2", "quote", @@ -5301,8 +5301,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-arith", "arrow-array", @@ -5335,8 +5335,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-arith", "arrow-array", @@ -5368,8 +5368,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "datafusion", "geo-traits", @@ -5383,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arc-swap", "arrow", @@ -5452,8 +5452,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-schema", @@ -5475,8 +5475,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow", "arrow-array", @@ -5516,8 +5516,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-schema", @@ -5531,8 +5531,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow", "async-trait", @@ -5546,8 +5546,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow", "arrow-ipc", @@ -5600,8 +5600,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -5615,8 +5615,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow", "arrow-array", @@ -5656,8 +5656,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "arrow-array", "arrow-schema", @@ -5670,8 +5670,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 7ec2b3538..cb67b9f8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index b9d3a2453..e2c0c319b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.6 + 13.0.0-beta.7 false 2.30.0 1.7 From 73346c0a2c45f0b934016629b154f897c9710eaa Mon Sep 17 00:00:00 2001 From: Lance Release Date: Sun, 20 Sep 2026 08:28:29 +0000 Subject: [PATCH 02/15] =?UTF-8?q?Bump=20version:=200.40.0-beta.3=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 314e75779..00c38dc79 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.3" +current_version = "0.40.0-beta.4" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index f7034c9dd..f4491ba13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.3" +version = "0.40.0-beta.4" dependencies = [ "ahash", "anyhow", @@ -5781,7 +5781,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.3" +version = "0.40.0-beta.4" dependencies = [ "arrow-array", "arrow-buffer", @@ -5806,7 +5806,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.3" +version = "0.40.0-beta.4" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index ee478561d..a877b25c4 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.3 + 0.40.0-beta.4 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index dda6c2e49..85d149ed0 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.3 + 0.40.0-beta.4 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index e2c0c319b..5e5068552 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.3 + 0.40.0-beta.4 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index bbc687a0b..05b336f04 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.3" +version = "0.40.0-beta.4" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index eaf211df2..d41265130 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index d51ab1418..a93727003 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 1cbab77b7..8d5843ec8 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 572e11db7..a3bb2e449 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 314c55e28..0c5e31824 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 79794ae7f..fdcc00e3b 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 6c0bcd929..02be1aba2 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 9ccc0885e..b7549d143 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.4", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 9714aff24..b2d7450e3 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.3" +version = "0.40.0-beta.4" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index a183b9e0d..57a21eee2 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.3" +version = "0.40.0-beta.4" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 3be29228e40b2ae7ece3cfaa1fad377217bc946b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 21 Sep 2026 20:33:50 +0800 Subject: [PATCH 03/15] chore: remove CLAUDE.md now that Claude reads AGENTS.md (#4238) ## Summary - Remove the `CLAUDE.md` agent instruction files at the repo root, `python/`, and `nodejs/`. - Claude now reads `AGENTS.md`, and these files were only symlinks to the existing `AGENTS.md` copies. ## Test plan - [x] Confirm `AGENTS.md` remains at the repo root, `python/`, and `nodejs/` - [x] Confirm no remaining in-repo references to `CLAUDE.md` Made with [Cursor](https://cursor.com) Co-authored-by: Cursor --- CLAUDE.md | 1 - nodejs/CLAUDE.md | 1 - python/CLAUDE.md | 1 - 3 files changed, 3 deletions(-) delete mode 120000 CLAUDE.md delete mode 120000 nodejs/CLAUDE.md delete mode 120000 python/CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3d8..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/nodejs/CLAUDE.md b/nodejs/CLAUDE.md deleted file mode 120000 index 47dc3e3d8..000000000 --- a/nodejs/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/python/CLAUDE.md b/python/CLAUDE.md deleted file mode 120000 index 47dc3e3d8..000000000 --- a/python/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file From 7e171f3abb94b85077d6c468fd755cd6a5850a84 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Mon, 21 Sep 2026 11:08:11 -0700 Subject: [PATCH 04/15] chore: update lance dependency to v13.0.0-beta.8 (#4242) Updates the Rust workspace Lance dependencies and Java lance-core to [v13.0.0-beta.8](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.8), refreshing Cargo.lock including the required goosefs-sdk 0.2.2 update. No source compatibility changes were required. Validation passed: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. --- Cargo.lock | 227 ++++++++++++++++++++++++++++++++++++--------------- Cargo.toml | 28 +++---- java/pom.xml | 2 +- 3 files changed, 174 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4491ba13..bb5f388cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1819,7 +1819,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2937,6 +2937,12 @@ dependencies = [ "sqlparser 0.62.0", ] +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] name = "debugid" version = "0.8.0" @@ -3081,7 +3087,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3309,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3499,6 +3505,65 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer-common" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +dependencies = [ + "anyhow", + "bytes", + "cfg-if 1.0.4", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +dependencies = [ + "anyhow", + "asyncband", + "bitflags 2.11.1", + "datasketches", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.17.1", + "itertools 0.15.0", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +dependencies = [ + "tokio", +] + [[package]] name = "frostem" version = "1.20260804.1" @@ -3523,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4004,18 +4069,23 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f521e3beacc809ced36a8aff4a468b1bf61302c8f4dca41c3f21ac0557c22ed" +checksum = "1e471f7bce40936cf2023855c7350e31e11b3f703eb7cb2ce73c9ffc451f87d2" dependencies = [ "arc-swap", "async-trait", "bytes", "dashmap", "fastrand", + "foyer-common", + "foyer-memory", "futures", "hostname", + "io-uring", "itoa", + "libc", + "lru", "prost", "rand 0.9.5", "serde", @@ -4468,7 +4538,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -4810,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5073,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arc-swap", "arrow", @@ -5146,8 +5216,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5169,7 +5239,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5183,7 +5253,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5192,8 +5262,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrayref", "crunchy", @@ -5203,8 +5273,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5241,8 +5311,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5273,8 +5343,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5291,8 +5361,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "proc-macro2", "quote", @@ -5301,8 +5371,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5335,8 +5405,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5368,8 +5438,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "datafusion", "geo-traits", @@ -5383,8 +5453,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arc-swap", "arrow", @@ -5452,8 +5522,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5475,8 +5545,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5488,6 +5558,7 @@ dependencies = [ "bytes", "chrono", "futures", + "goosefs-sdk", "http 1.5.0", "io-uring", "lance-core", @@ -5498,6 +5569,7 @@ dependencies = [ "object_store 0.14.1", "object_store_opendal", "opendal", + "opendal-service-goosefs", "path_abs", "pin-project", "prost", @@ -5516,8 +5588,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5531,8 +5603,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "async-trait", @@ -5546,8 +5618,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-ipc", @@ -5600,8 +5672,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5615,8 +5687,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5656,8 +5728,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5670,8 +5742,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.7#ba4192424e7a871a448431f247366df2f3609ed7" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "frostem", "icu_segmenter", @@ -6207,6 +6279,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.6" @@ -6280,6 +6361,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.16" @@ -6540,7 +6631,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6669,7 +6760,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -8043,8 +8134,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -8063,7 +8154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8285,7 +8376,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.45", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -8323,7 +8414,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -9123,7 +9214,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9194,7 +9285,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9741,7 +9832,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -9753,7 +9844,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -9776,7 +9867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9895,7 +9986,7 @@ dependencies = [ "cfg-if 1.0.4", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10187,7 +10278,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11208,7 +11299,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cb67b9f8c..eb691ac48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.7", default-features = false, "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.7", "tag" = "v13.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 5e5068552..a42d31540 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.7 + 13.0.0-beta.8 false 2.30.0 1.7 From df2e785df73f75cc17c87485d5345969056837dd Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 21 Sep 2026 11:51:44 -0700 Subject: [PATCH 05/15] feat: retire a Function binding when its output columns are dropped (#4231) Dropping a column a Function binding writes was refused outright, which left a bad declaration unrecoverable: the binding is immutable, there is no rebind, and so the column could never be filled again. A drop that names every output of a binding now retires it instead. A binding lives in three places -- the envelope in schema metadata, a `computed_column.*` marker on each output field, and the columns themselves -- and readers cross-check the first two, so removing one and leaving the others is a table that refuses every write. The retirement therefore commits in two steps, each landing a state that stands on its own: one `UpdateConfig` rewrites the envelope and clears the markers together, leaving the outputs as ordinary columns holding their last values, and the drop follows. Interrupted between them, the columns are still there to be dropped again. Folding the metadata into the drop's own `Project` was the obvious alternative and does not work: the transaction proto records `Project` as fields alone, so the edit would survive only in the writer's memory. Naming one output of a multi-output binding is refused and names the missing siblings, since one refresh writes them in one commit. A multi-output binding's hidden `__function_assignment_*` column goes with it. Inputs stay protected while a surviving binding reads them, and drop in the request that retires the last one. --- rust/lancedb/src/table/computed_columns.rs | 190 ++++++++- rust/lancedb/src/table/schema_evolution.rs | 447 +++++++++++++++++++-- 2 files changed, 596 insertions(+), 41 deletions(-) diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 23a870b42..679928cd3 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -418,23 +418,18 @@ pub(crate) fn ensure_not_function_bound>( touched: impl IntoIterator, ) -> Result<()> { ensure_supported_function_metadata(schema)?; - let mut protected = BTreeSet::new(); - for binding in function_bindings(schema)? { - for input in binding.inputs() { - protected.insert(field_root(&input.field_path)?); - } - protected.extend( - binding - .outputs() - .iter() - .map(|output| output.output_name.clone()), - ); - protected.extend( - binding - .assignment() - .map(|assignment| assignment.output_name.clone()), - ); - } + ensure_not_bound_by(&function_bindings(schema)?, operation, touched) +} + +/// [`ensure_not_function_bound`] against an explicit set of bindings, for a +/// caller that is retiring some of the schema's own in the same operation and +/// must be checked against what survives it. +pub(crate) fn ensure_not_bound_by>( + bindings: &[FunctionBinding], + operation: &str, + touched: impl IntoIterator, +) -> Result<()> { + let protected = protected_roots(bindings)?; if protected.is_empty() { return Ok(()); } @@ -452,6 +447,129 @@ pub(crate) fn ensure_not_function_bound>( Ok(()) } +/// Every column `bindings` depend on, inputs taken at their root. +fn protected_roots(bindings: &[FunctionBinding]) -> Result> { + let mut protected = BTreeSet::new(); + for binding in bindings { + for input in binding.inputs() { + protected.insert(field_root(&input.field_path)?); + } + protected.extend( + binding + .outputs() + .iter() + .map(|output| output.output_name.clone()), + ); + protected.extend( + binding + .assignment() + .map(|assignment| assignment.output_name.clone()), + ); + } + Ok(protected) +} + +/// What a drop must do besides removing columns, so that the Function +/// bindings it covers are retired rather than stranded. +#[derive(Debug, Default)] +pub struct FunctionUnbinding { + /// The bindings the drop leaves in place. Later columns in the same + /// operation are checked against these, not against the schema's. + pub retained: Vec, + /// Assignment columns of the retired bindings that the caller did not + /// name. Internal bookkeeping columns, so the drop adds them itself. + pub assignment_columns: Vec, + /// Columns whose `computed_column.*` declaration metadata the unbind + /// commit clears. + pub cleared_columns: Vec, + /// The new value of [`FUNCTION_BINDINGS_META_KEY`]; `None` deletes it. + pub bindings_metadata: Option, +} + +impl FunctionUnbinding { + /// True when the drop retires nothing and is an ordinary column drop. + pub fn is_noop(&self) -> bool { + self.cleared_columns.is_empty() + } + + /// Refuse `operation` on a path that a binding surviving this drop still + /// reads or writes. The retired ones no longer protect anything. + pub fn ensure_retained_unaffected>( + &self, + operation: &str, + touched: impl IntoIterator, + ) -> Result<()> { + ensure_not_bound_by(&self.retained, operation, touched) + } +} + +/// Plan the retirement of every Function binding whose outputs `columns` +/// covers. +/// +/// Naming one output of a binding means naming all of them: the outputs of +/// one Function are written by one refresh in one commit, so a binding that +/// kept some of them would have no coherent shape to write. A partial drop is +/// refused and names the siblings that are missing. +pub fn plan_function_unbinding( + schema: &ArrowSchema, + columns: &[&str], +) -> Result { + ensure_supported_function_metadata(schema)?; + let named = columns + .iter() + .map(|column| whole_column(column)) + .collect::>>>()? + .into_iter() + .flatten() + .collect::>(); + + let mut plan = FunctionUnbinding::default(); + for binding in function_bindings(schema)? { + let outputs = binding + .outputs() + .iter() + .map(|output| output.output_name.clone()) + .collect::>(); + let Some(dropped) = outputs.iter().find(|name| named.contains(name.as_str())) else { + plan.retained.push(binding); + continue; + }; + let missing = outputs + .iter() + .filter(|name| !named.contains(name.as_str())) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(Error::InvalidInput { + message: format!( + "dropping Function output '{dropped}' must drop every output of its \ + binding; the same request must also name {}", + missing.join(", ") + ), + }); + } + plan.cleared_columns.extend(outputs); + if let Some(assignment) = binding.assignment() { + plan.cleared_columns.push(assignment.output_name.clone()); + if !named.contains(assignment.output_name.as_str()) { + plan.assignment_columns.push(assignment.output_name.clone()); + } + } + } + + // Re-encoding an untouched set would rewrite bytes a stricter reader + // compares against its own encoding, so leave the key alone when the drop + // retires nothing. + if !plan.is_noop() { + plan.bindings_metadata = if plan.retained.is_empty() { + None + } else { + Some(function_bindings_metadata(&plan.retained)?) + }; + } + Ok(plan) +} + /// Read a field's computed-column declaration, if it carries one. /// /// A field flagged computed but carrying no kind, or a SQL one missing its @@ -1320,7 +1438,30 @@ pub(crate) fn plan_function_application( /// Paths are compared at their root: a declaration reading `metadata` is /// invalidated by a change to `metadata.age` just as surely. pub(crate) fn ensure_not_an_input(schema: &SchemaRef, paths: &[&str]) -> Result<()> { + ensure_not_an_input_of(schema, paths, &[]) +} + +/// [`ensure_not_an_input`] where `removed` names columns whose own +/// declarations the same operation deletes. A reader that is going away +/// cannot be stranded by dropping what it reads, so a group drops together. +pub(crate) fn ensure_not_an_input_of( + schema: &SchemaRef, + paths: &[&str], + removed: &[&str], +) -> Result<()> { + // By identity, not spelling: a quoted path names the same column, while + // a path *into* one removes no declaration at all. + let removed = removed + .iter() + .map(|path| whole_column(path)) + .collect::>>()? + .into_iter() + .flatten() + .collect::>(); for declaration in computed_columns(schema) { + if removed.contains(&declaration.name) { + continue; + } // The expression, not stored inputs, is the source of truth; an // expression that no longer parses proves nothing, so refuse. let inputs = match &declaration.kind { @@ -1583,6 +1724,21 @@ pub(crate) fn field_root(path: &str) -> Result { }) } +/// The column a path names when it names the whole column, and `None` when it +/// addresses a field inside one. Retiring a binding takes the whole output; +/// a path into one is left to the ordinary guard to refuse. +fn whole_column(path: &str) -> Result> { + let mut segments = parse_field_path(path) + .map_err(|e| Error::InvalidInput { + message: format!("invalid column path '{path}': {e}"), + })? + .into_iter(); + let column = segments.next().ok_or_else(|| Error::InvalidInput { + message: format!("column path '{path}' is empty"), + })?; + Ok(segments.next().is_none().then_some(column)) +} + /// [`field_root`], falling back to the text before the first dot for a /// spelling lance would not resolve. pub(crate) fn root(path: &str) -> String { diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 796db09bd..4d0e2815d 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -9,7 +9,8 @@ //! - [`drop_columns`](execute_drop_columns): Remove columns from the table use arrow_schema::Schema as ArrowSchema; -use lance::dataset::{ColumnAlteration, NewColumnTransform}; +use lance::dataset::transaction::{Operation, Transaction, UpdateMap, UpdateMapEntry}; +use lance::dataset::{ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -240,14 +241,96 @@ pub(crate) async fn execute_drop_columns( table.dataset.ensure_mutable()?; let mut dataset = (*table.dataset.get().await?).clone(); let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema())); - computed_columns::ensure_not_function_bound(schema.as_ref(), "schema evolution", columns)?; - computed_columns::ensure_not_an_input(&schema, columns)?; - dataset.drop_columns(columns).await?; + let unbinding = computed_columns::plan_function_unbinding(schema.as_ref(), columns)?; + + let mut names = columns.iter().map(|c| c.to_string()).collect::>(); + names.extend(unbinding.assignment_columns.iter().cloned()); + let dropped = names.iter().map(String::as_str).collect::>(); + computed_columns::ensure_not_bound_by(&unbinding.retained, "schema evolution", &dropped)?; + computed_columns::ensure_not_an_input_of(&schema, &dropped, &dropped)?; + + if !unbinding.is_noop() { + // The retirement commits before the projection, so the whole + // projection has to be known valid against this revision first -- + // otherwise the binding is gone and the columns are not. Lance plans + // it rather than this repeating the rules: dropping a struct's last + // child removes the struct, and data files that disagree on metadata + // semantics are refused, neither of which an approximation here would + // get right. + dataset.plan_drop_columns(&dropped)?; + dataset = commit_function_unbinding(dataset, &unbinding).await?; + } + dataset.drop_columns(&dropped).await?; let version = dataset.version().version; table.dataset.update(dataset); Ok(DropColumnsResult { version }) } +/// Retire the bindings a drop covers, in the commit before it. +/// +/// A binding lives in three places -- the schema-metadata envelope, each +/// output's `computed_column.*` field metadata, and the columns themselves -- +/// and readers cross-check the first two, so a state with one of them removed +/// is a table that refuses every write. Clearing both metadata halves in one +/// `UpdateConfig` leaves the outputs as ordinary columns holding their last +/// values: valid on its own, and the drop that follows is then an ordinary +/// drop. Interrupted in between, the columns are still there to be dropped +/// again. +/// +/// Not folded into the drop's own `Operation::Project`, though it carries a +/// schema: the transaction proto records `Project` as fields alone, so the +/// metadata edit would survive only in this writer's memory. +async fn commit_function_unbinding( + dataset: Dataset, + unbinding: &computed_columns::FunctionUnbinding, +) -> Result { + let bindings = UpdateMap { + update_entries: vec![UpdateMapEntry { + key: computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(), + value: unbinding.bindings_metadata.clone(), + }], + replace: false, + }; + let mut field_metadata_updates = HashMap::new(); + for column in &unbinding.cleared_columns { + let field = dataset + .schema() + .field(column) + .ok_or_else(|| Error::InvalidInput { + message: format!("Function output '{column}' does not exist in the table"), + })?; + let cleared = field + .metadata + .keys() + .filter(|key| computed_columns::is_declaration_key(key)) + .map(|key| UpdateMapEntry { + key: key.clone(), + value: None, + }) + .collect::>(); + field_metadata_updates.insert( + field.id, + UpdateMap { + update_entries: cleared, + replace: false, + }, + ); + } + let transaction = Transaction::new( + dataset.manifest.version, + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: None, + schema_metadata_updates: Some(bindings), + field_metadata_updates, + }, + None, + ); + Ok(CommitBuilder::new(std::sync::Arc::new(dataset)) + .execute(transaction) + .await?) +} + /// Internal implementation of the update field metadata logic. /// /// Merges or replaces per-field metadata, addressing fields by dot-path. @@ -330,8 +413,9 @@ mod tests { use crate::query::{ExecutableQuery, QueryBase, Select}; use crate::table::NewColumnTransform; use crate::table::computed_columns::{ - FUNCTION_BINDINGS_META_KEY, ensure_supported_function_metadata, function_bindings, - function_bindings_metadata, function_computed_column_metadata, + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, FUNCTION_BINDINGS_META_KEY, + ensure_supported_function_metadata, function_bindings, function_bindings_metadata, + function_computed_column_metadata, is_declaration_key, }; use crate::{Error, Table}; use std::collections::HashMap; @@ -351,38 +435,71 @@ mod tests { ) .unwrap(); let table = conn.create_table("bound", batch).execute().await.unwrap(); - let binding = FunctionBinding::from_json(include_str!( + stamp_bindings(&table, &[text_features_binding("")]).await; + table + } + + /// The fixture binding, its id and output names suffixed so a table can + /// carry more than one. + fn text_features_binding(suffix: &str) -> FunctionBinding { + let raw = include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" - )) - .unwrap(); + ); + if suffix.is_empty() { + return FunctionBinding::from_json(raw).unwrap(); + } + FunctionBinding::from_json( + &raw.replace("fb_01K3TEXT", &format!("fb_01K3TEXT{suffix}")) + .replace("search_text", &format!("search_text{suffix}")) + .replace("search_token_count", &format!("search_token_count{suffix}")), + ) + .unwrap() + } + + /// Stamp bindings the way the server does it, since no local path + /// declares one: the envelope on the schema, a declaration marker on + /// every output field. + async fn stamp_bindings(table: &Table, bindings: &[FunctionBinding]) { let native = table.as_native().unwrap(); let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); dataset .update_schema_metadata(vec![( FUNCTION_BINDINGS_META_KEY.to_string(), - Some(function_bindings_metadata(std::slice::from_ref(&binding)).unwrap()), + Some(function_bindings_metadata(bindings).unwrap()), )]) .await .unwrap(); let inputs = ["title".to_string(), "body".to_string()]; - let outputs = binding - .outputs() + let outputs = bindings .iter() - .map(|output| { - ( - dataset.schema().field(&output.output_name).unwrap().id as u32, - function_computed_column_metadata( - binding.binding_id(), - output.output_ordinal, - &inputs, - ), - ) + .flat_map(|binding| { + let assignment = binding.assignment().map(|assignment| { + ( + assignment.output_name.clone(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + ) + }); + binding + .outputs() + .iter() + .map(|output| (output.output_name.clone(), output.output_ordinal)) + .chain(assignment) + .map(|(name, ordinal)| { + ( + dataset.schema().field(&name).unwrap().id as u32, + function_computed_column_metadata( + binding.binding_id(), + ordinal, + &inputs, + ), + ) + }) + .collect::>() }) .collect::>(); dataset.replace_field_metadata(outputs).await.unwrap(); native.dataset.update(dataset); ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); - table } fn metadata_update(path: &str) -> FieldMetadataUpdate { @@ -438,6 +555,18 @@ mod tests { ); } + fn incomplete_group(err: Error) { + assert!( + matches!(&err, Error::InvalidInput { message } + if message.contains("must drop every output of its binding")), + "{err:?}" + ); + } + + async fn bindings_of(table: &Table) -> Vec { + function_bindings(table.schema().await.unwrap().as_ref()).unwrap() + } + /// Every schema-evolution door refuses a column a binding reads or /// writes, including a rename onto one. #[tokio::test] @@ -445,7 +574,14 @@ mod tests { let table = bound_table().await; let version = table.version().await.unwrap(); for column in ["title", "body", "search_text", "search_token_count"] { - bound(table.drop_columns(&[column]).await.unwrap_err()); + // A lone drop of an output is refused too, but for its siblings + // rather than for the binding; see the retirement tests below. + let err = table.drop_columns(&[column]).await.unwrap_err(); + if column.starts_with("search_") { + incomplete_group(err); + } else { + bound(err); + } bound( table .alter_columns(&[ColumnAlteration::new(column.into()).rename("moved".into())]) @@ -500,7 +636,12 @@ mod tests { let table = bound_table().await; let version = table.version().await.unwrap(); for path in ["`title`", "`search_text`", "`title`.nested"] { - bound(table.drop_columns(&[path]).await.unwrap_err()); + let err = table.drop_columns(&[path]).await.unwrap_err(); + if path == "`search_text`" { + incomplete_group(err); + } else { + bound(err); + } bound( table .alter_columns(&[ColumnAlteration::new(path.into()).set_nullable(false)]) @@ -1038,6 +1179,264 @@ mod tests { assert_eq!(drop_result.version, v4); } + // Retiring a Function binding by dropping its outputs (ENT-2669). + + /// The whole output set goes, and the binding with it: no envelope, no + /// declaration markers, and the inputs it held are ordinary again. + #[tokio::test] + async fn dropping_every_output_retires_the_binding() { + let table = bound_table().await; + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + + let schema = table.schema().await.unwrap(); + assert!(schema.field_with_name("search_text").is_err()); + assert!(schema.field_with_name("search_token_count").is_err()); + assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + assert!(bindings_of(&table).await.is_empty()); + + // What the binding used to protect is now ordinary schema. + table.drop_columns(&["title"]).await.unwrap(); + table + .update_field_metadata(&[metadata_update("body")]) + .await + .unwrap(); + ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); + } + + /// One output of a multi-output binding cannot go alone: its siblings are + /// written by the same refresh in the same commit. + #[tokio::test] + async fn dropping_part_of_an_output_group_names_the_missing_siblings() { + let table = bound_table().await; + let version = table.version().await.unwrap(); + for (named, missing) in [ + ("search_text", "search_token_count"), + ("search_token_count", "search_text"), + ] { + let err = table.drop_columns(&[named]).await.unwrap_err(); + let Error::InvalidInput { message } = &err else { + panic!("{err:?}"); + }; + assert!( + message.contains("must drop every output of its binding") + && message.contains(missing), + "{message}" + ); + } + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(bindings_of(&table).await.len(), 1); + } + + /// An input is protected by the binding, not by its own name, so it drops + /// in the request that retires the binding reading it. + #[tokio::test] + async fn an_input_drops_with_the_binding_that_reads_it() { + let table = bound_table().await; + bound(table.drop_columns(&["title"]).await.unwrap_err()); + + table + .drop_columns(&["title", "body", "search_text", "search_token_count"]) + .await + .unwrap(); + let schema = table.schema().await.unwrap(); + assert_eq!(schema.fields().len(), 1); + assert!(schema.field_with_name("spare").is_ok()); + assert!(bindings_of(&table).await.is_empty()); + } + + /// Lance resolves a quoted spelling to the same field, so it retires the + /// same binding rather than slipping past the group rule. + #[tokio::test] + async fn a_quoted_output_spelling_retires_the_same_binding() { + let table = bound_table().await; + incomplete_group(table.drop_columns(&["`search_text`"]).await.unwrap_err()); + table + .drop_columns(&["`search_text`", "`search_token_count`"]) + .await + .unwrap(); + assert!(bindings_of(&table).await.is_empty()); + } + + /// The unbind lands before the drop, and on its own: at that version the + /// outputs are still there as plain columns, with no binding and no + /// declaration metadata left pointing at one. A crash in between leaves + /// that state, which every later write must accept. + #[tokio::test] + async fn the_unbind_commit_leaves_a_valid_table_before_the_drop() { + let table = bound_table().await; + let before = table.version().await.unwrap(); + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + assert_eq!(table.version().await.unwrap(), before + 2); + + table.checkout(before + 1).await.unwrap(); + let schema = table.schema().await.unwrap(); + assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + for column in ["search_text", "search_token_count"] { + let field = schema.field_with_name(column).unwrap(); + assert!( + !field.metadata().keys().any(|key| is_declaration_key(key)), + "{column}: {:?}", + field.metadata() + ); + } + ensure_supported_function_metadata(&schema).unwrap(); + assert!(function_bindings(&schema).unwrap().is_empty()); + } + + /// A table carrying two bindings retires only the one whose outputs the + /// drop names; the other keeps its envelope entry and its protection. + #[tokio::test] + async fn retiring_one_binding_leaves_the_others_standing() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + ("search_text_2", Utf8, ["a b"]), + ("search_token_count_2", Int64, [2]) + ) + .unwrap(); + let table = conn.create_table("two", batch).execute().await.unwrap(); + stamp_bindings( + &table, + &[text_features_binding(""), text_features_binding("_2")], + ) + .await; + + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + let remaining = bindings_of(&table).await; + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].binding_id(), "fb_01K3TEXT_2"); + // The survivor still protects the inputs it shared with the retired one. + bound(table.drop_columns(&["title"]).await.unwrap_err()); + // And its own outputs are still a group. + incomplete_group(table.drop_columns(&["search_text_2"]).await.unwrap_err()); + } + + /// A multi-output binding also owns a hidden `__function_assignment_*` + /// column, which the caller never names. Retiring the binding has to take + /// it: leaving it behind orphans a column nothing can fill or explain. + #[tokio::test] + async fn retiring_a_binding_drops_its_assignment_column() { + const ASSIGNMENT: &str = "__function_assignment_fb_01K3TEXT"; + + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + (ASSIGNMENT, Boolean, [Some(true)]), + ("spare", Int32, [1]) + ) + .unwrap(); + let table = conn + .create_table("assigned", batch) + .execute() + .await + .unwrap(); + + let raw = include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + ) + .replace( + r#""future_binding""#, + &format!( + r#""assignment": {{"output_name": "{ASSIGNMENT}", "output_field_id": -1}}, "future_binding""# + ), + ) + // The assignment column is a physical sibling, so it belongs to the + // binding's output schema too -- the server writes it that way. + .replace( + r#"{"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}"#, + &format!( + r#"{{"name": "search_token_count", "nullable": true, "type": {{"type": "int64"}}}}, {{"name": "{ASSIGNMENT}", "nullable": true, "type": {{"type": "bool"}}}}"# + ), + ); + let binding = FunctionBinding::from_json(&raw).unwrap(); + assert!(binding.assignment().is_some(), "fixture must carry one"); + stamp_bindings(&table, std::slice::from_ref(&binding)).await; + + // The assignment column is the binding's, so it is protected too... + bound(table.drop_columns(&[ASSIGNMENT]).await.unwrap_err()); + + // ...and goes with the outputs without the caller naming it. + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + let schema = table.schema().await.unwrap(); + assert!( + schema.field_with_name(ASSIGNMENT).is_err(), + "the assignment column outlived its binding: {:?}", + schema.fields().iter().map(|f| f.name()).collect::>() + ); + assert!(bindings_of(&table).await.is_empty()); + ensure_supported_function_metadata(&schema).unwrap(); + } + + /// An input error must not cost the binding. The retirement commits + /// first, so a projection that could never succeed is refused before it, + /// leaving the table exactly as it was. + #[rstest::rstest] + #[case::unknown_column( + &["search_text", "search_token_count", "nope"], + "does not exist" + )] + #[case::every_column( + &["title", "body", "search_text", "search_token_count", "spare"], + "Cannot drop all columns" + )] + #[case::every_column_quoted( + &["`title`", "`body`", "`search_text`", "`search_token_count`", "`spare`"], + "Cannot drop all columns" + )] + #[tokio::test] + async fn an_invalid_drop_leaves_the_binding_in_place( + #[case] columns: &[&str], + #[case] expected: &str, + ) { + let table = bound_table().await; + let version = table.version().await.unwrap(); + + let err = table.drop_columns(columns).await.unwrap_err(); + let Error::InvalidInput { message } = &err else { + panic!("{err:?}"); + }; + assert!(message.contains(expected), "{message}"); + + // Read durable state, not this handle: the error path never calls + // `dataset.update`, so the cached handle would report the old version + // even if the unbind had committed. + table.checkout_latest().await.unwrap(); + assert_eq!(table.version().await.unwrap(), version); + let schema = table.schema().await.unwrap(); + assert!(schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + assert_eq!(bindings_of(&table).await.len(), 1); + for column in ["search_text", "search_token_count"] { + assert!( + schema + .field_with_name(column) + .unwrap() + .metadata() + .keys() + .any(|key| is_declaration_key(key)), + "{column} lost its declaration metadata" + ); + } + ensure_supported_function_metadata(&schema).unwrap(); + } + #[tokio::test] async fn test_update_field_metadata() { let conn = connect("memory://").execute().await.unwrap(); From a8aa19f513b05600c065176e5effd04d459ff14d Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 21 Sep 2026 18:52:46 +0000 Subject: [PATCH 06/15] =?UTF-8?q?Bump=20version:=200.40.0-beta.4=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 00c38dc79..2dfbe973d 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.4" +current_version = "0.40.0-beta.5" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index bb5f388cd..9905e8b86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5756,7 +5756,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" dependencies = [ "ahash", "anyhow", @@ -5853,7 +5853,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" dependencies = [ "arrow-array", "arrow-buffer", @@ -5878,7 +5878,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a877b25c4..f6e21c69e 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.4 + 0.40.0-beta.5 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 85d149ed0..e38de7a79 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.4 + 0.40.0-beta.5 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a42d31540..845d1ea07 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.4 + 0.40.0-beta.5 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 05b336f04..e2357dfc3 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index d41265130..b517a7b8a 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index a93727003..25c6698c3 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 8d5843ec8..c46f40f96 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index a3bb2e449..fc030ef53 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 0c5e31824..4f23a3261 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index fdcc00e3b..0594f5610 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 02be1aba2..2c819c574 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index b7549d143..b35b5eb28 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.4", + "version": "0.40.0-beta.5", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index b2d7450e3..8fcdf8914 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 57a21eee2..ebec41318 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.4" +version = "0.40.0-beta.5" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From bd94fc3572975ef1d5fbb2815005177317afbe3b Mon Sep 17 00:00:00 2001 From: Ryan Green Date: Mon, 21 Sep 2026 20:07:24 -0230 Subject: [PATCH 07/15] feat(remote): support add_columns with schema in remote client (#4244) Support adding column with pyarrow schema in remote client. Previously this raised an error. This achieves parity with local client. Server-side implementation is already complete. `RemoteTable::add_columns` matched only `SqlExpressions` and refused everything else, so `add_columns(pa.Field | List[pa.Field] | pa.Schema)` reached a remote table as `NotSupported`. Every layer above was already in place: the Python and Node surfaces accept a schema, both bindings build `NewColumnTransform::AllNulls` from it, and the Function-binding guard already reads that variant's column names. Only the arm that turns it into a request was missing. Send the schema as an Arrow IPC schema message under `application/vnd.apache.arrow.stream`, which is what the server takes. It is also the only encoding that round-trips a field whole: the JSON type representations either hide decimal precision in a length field or drop a timestamp's unit and timezone. With no JSON envelope the branch rides the query string, as it does for the other binary-bodied endpoints. The response handling is now shared by both arms rather than living inside the SQL one, so the new path gets the same schema-cache invalidation, write-version tracking, and old-server empty-body fallback. Also widen the sync `RemoteTable.add_columns` annotation to match `Table.add_columns`. It delegates to the async table, so a schema already worked at runtime; only the signature and its docs disagreed. Co-authored-by: Claude Opus 5 (1M context) --- python/python/lancedb/remote/table.py | 3 + rust/lancedb/src/remote/table.rs | 164 +++++++++++++++++++++----- 2 files changed, 138 insertions(+), 29 deletions(-) diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index a9fc06bdd..2a6512fc5 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -982,6 +982,9 @@ class RemoteTable(Table): self, transforms: Dict[str, str | FunctionApplication] | FunctionApplication + | pa.Field + | List[pa.Field] + | pa.Schema | None = None, *, computed: Dict[str, str] | None = None, diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 68c8409a9..0c51f50ae 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1486,6 +1486,20 @@ fn resolve_arrow_ipc_framing( }) } +/// An Arrow IPC stream carrying `schema` and no batches. +/// +/// The body of an all-null `add_columns`: the server reads the fields to +/// add straight off the schema message, so nothing but the field list is +/// worth sending. +fn write_ipc_schema(schema: &arrow_schema::Schema) -> Result> { + let mut body = Vec::new(); + { + let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut body, schema)?; + writer.finish()?; + } + Ok(body) +} + /// Strip media-type parameters before matching against the Arrow content types. fn base_media_type(content_type: &str) -> &str { match content_type.split_once(';') { @@ -3325,7 +3339,8 @@ impl BaseTable for RemoteTable { "schema evolution", crate::table::schema_evolution::new_column_names(&transforms), )?; - match transforms { + let path = format!("/v1/table/{}/add_columns/", self.identifier); + let request = match transforms { NewColumnTransform::SqlExpressions(expressions) => { let body = expressions .into_iter() @@ -3338,40 +3353,55 @@ impl BaseTable for RemoteTable { .collect::>(); let mut body = serde_json::json!({ "new_columns": body }); self.apply_branch_body(&mut body); - let request = self - .client - .post(&format!("/v1/table/{}/add_columns/", self.identifier)) - .json(&body); - let freshness_request = self.snapshot_freshness_headers(); - let (request_id, response) = self - .send_with_freshness(request, true, freshness_request) - .await?; - let response = self.check_table_response(&request_id, response).await?; - let body = response.text().await.err_to_http(request_id.clone())?; - - if body.trim().is_empty() { - // Backward compatible with old servers - return Ok(AddColumnsResult { version: 0 }); - } - - let result: AddColumnsResult = - serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse add_columns response: {}", e).into(), - request_id, - status_code: None, - })?; - - self.invalidate_schema_cache(); - self.track_write_version(freshness_request, result.version); - - Ok(result) + self.client.post(&path).json(&body) + } + // Every field becomes a column that reads null for existing + // rows. The schema goes over the wire as Arrow IPC rather than + // a JSON description of the types: that is what the server + // takes, and it is the only encoding that round-trips a field + // whole, including decimal precision and a timestamp's unit and + // timezone. With no JSON envelope the branch rides the query + // string, as it does for the other binary-bodied endpoints. + NewColumnTransform::AllNulls(schema) => { + let body = write_ipc_schema(schema.as_ref())?; + self.apply_branch_query( + self.client + .post(&path) + .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE) + .body(body), + ) } _ => { return Err(Error::NotSupported { - message: "Only SQL expressions are supported for adding columns".into(), + message: "Only SQL expressions and all-null column schemas are supported \ + for adding columns" + .into(), }); } + }; + + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; + let response = self.check_table_response(&request_id, response).await?; + let body = response.text().await.err_to_http(request_id.clone())?; + + if body.trim().is_empty() { + // Backward compatible with old servers + return Ok(AddColumnsResult { version: 0 }); } + + let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse add_columns response: {}", e).into(), + request_id, + status_code: None, + })?; + + self.invalidate_schema_cache(); + self.track_write_version(freshness_request, result.version); + + Ok(result) } async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result { @@ -7913,6 +7943,82 @@ mod tests { assert_eq!(result.version, if old_server { 0 } else { 43 }); } + /// A pyarrow schema goes over the wire as an Arrow IPC schema under an + /// Arrow content type, not as a JSON description of the types, so a + /// field arrives exactly as sent. The branch has no JSON envelope to + /// ride in and goes on the query string. + #[tokio::test] + async fn test_add_columns_sends_an_arrow_schema_for_all_nulls() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => simple_describe_response(), + "/v1/table/my_table/add_columns/" => { + assert_eq!(request.method(), "POST"); + assert_eq!( + request.headers().get("Content-Type").unwrap(), + ARROW_STREAM_CONTENT_TYPE + ); + assert_eq!( + request.url().query_pairs().find(|(k, _)| k == "branch"), + None, + "the main branch sends no branch parameter" + ); + + // The payload decodes back to the exact schema, types and + // field metadata included. + let body = request.body().unwrap().as_bytes().unwrap(); + let reader = + arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(body), None) + .unwrap(); + let sent = reader.schema(); + assert_eq!(sent.fields().len(), 2); + assert_eq!(sent.field(0).name(), "ts"); + assert_eq!( + sent.field(0).data_type(), + &DataType::Timestamp( + arrow_schema::TimeUnit::Nanosecond, + Some("America/New_York".into()) + ) + ); + assert!(sent.field(0).is_nullable()); + assert_eq!(sent.field(1).name(), "price"); + assert_eq!(sent.field(1).data_type(), &DataType::Decimal128(38, 10)); + assert_eq!( + sent.field(1).metadata().get("unit").map(String::as_str), + Some("cents") + ); + + http::Response::builder() + .status(200) + .body(r#"{"version": 7}"#.to_string()) + .unwrap() + } + path => panic!("Unexpected request path: {path}"), + }); + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "ts", + DataType::Timestamp( + arrow_schema::TimeUnit::Nanosecond, + Some("America/New_York".into()), + ), + true, + ), + Field::new("price", DataType::Decimal128(38, 10), true).with_metadata( + std::collections::HashMap::from([("unit".to_string(), "cents".to_string())]), + ), + ])); + + let result = table + .add_columns() + .transform(NewColumnTransform::AllNulls(schema)) + .execute() + .await + .unwrap(); + + assert_eq!(result.version, 7); + } + /// A declaration is sent as `{name, computed}` for the server to plan; the /// client never types the expression itself. #[tokio::test] From ff757b74fdeabe7aed74b9d1d117595836c67c07 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 22 Sep 2026 14:59:52 +0800 Subject: [PATCH 08/15] fix(python): reduce Windows release wheel size (#4241) The Windows wheel for 0.39.0 exceeded PyPI's 100 MiB file-size limit, preventing the original release upload. Restore the repository's release profile (`fat` LTO, 1 codegen unit) by removing the Windows-only `thin`/16 overrides introduced in #3716. Keep `rust-lld` as the linker. This recovers wheel-size headroom at the cost of the longer fat-LTO build. The already-published 0.39.0 Windows wheel was recovered separately by recompressing the original artifact; this change addresses the build configuration for future releases. Fixes #4239. ### Windows size comparison Built the same v0.38.0 source (`8c68e0c619f2b1febe92e51a29d72c968d24a5c9`) twice on one AWS `m7i.4xlarge` Windows Server 2025 machine, changing only the LTO/codegen-unit overrides: | Artifact | thin LTO / 16 units | fat LTO / 1 unit | | --- | ---: | ---: | | Installable Windows wheel | 104,129,447 bytes (99.31 MiB) | 73,875,399 bytes (70.45 MiB) | | Uncompressed native module | 309,969,408 bytes (295.61 MiB) | 206,609,408 bytes (197.04 MiB) | The thin/16 configuration increased wheel size by **40.95%** and native-module size by **50.03%** relative to fat/1. The compressed native module accounts for all but 3 bytes of the wheel increase. Both builds used Rust 1.97.0, maturin 1.12.4, Python 3.13.5, MSVC 14.44.35207, Windows SDK 10.0.26100.0, rust-lld, static CRT, default features, and `maturin build --release --strip --locked --verbose`. They ran sequentially with separate empty target directories and identical locked third-party dependencies. The tag's stale workspace package versions were normalized once before both builds. This controlled pair used VS2022 Build Tools; the historical GitHub runner used VS2026. Both wheels passed ZIP/RECORD integrity checks and installed successfully. Native smoke checks covered import, database creation, row count, nearest-vector query, and reopening the database. This establishes the combined configuration effect; it does not isolate LTO mode from codegen-unit count or establish a runtime-performance difference. The figures above are the v0.38.0 reproduction, **not measurements of this PR head**. The existing PyPI Publish pull-request workflow rebuilds the current revision without publishing; its result is pending. Workflow validation passed with `actionlint -shellcheck=`. Full actionlint reports the same three pre-existing ShellCheck diagnostics in the unchanged repository-selection step as on `main`. --- .github/workflows/pypi-publish.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index b80c1b019..5f45ba68c 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -129,12 +129,8 @@ jobs: # link.exe is single-threaded and the long pole on Windows builds. Use # rustc's bundled lld-link instead. CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld - # Fat LTO of the cdylib is single-threaded and the peak-memory step of the - # build. ThinLTO parallelizes it across the runner's cores, at some cost - # to runtime performance on our least performance-sensitive platform. - # Matches what the nodejs Windows builds already do in npm-publish.yml. - CARGO_PROFILE_RELEASE_LTO: thin - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 + # Keep the repository's fat LTO / 1 codegen unit release profile to leave + # headroom below PyPI's file-size limit for Windows wheels. steps: - uses: actions/checkout@v6 with: From 518d7ff1dd7aec79fb408df4e59ce01abd44b7fd Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 22 Sep 2026 15:13:51 +0800 Subject: [PATCH 09/15] fix: keep REST server adapter out of production dependencies (#4240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The production `remote` client talks to an existing namespace over HTTP. It needs `lance-namespace-impls/rest`, not the REST *server* adapter. `RestAdapter` is only used by `cfg(test)` integration tests that stand up an in-process server. Enabling `rest-adapter` on the production `remote` feature pulled that server stack, including Axum 0.7, into default Python wheels. This keeps `rest` on `remote` and moves `rest-adapter` to a `lance-namespace-impls` dev-dependency so those tests still compile and run. Cargo resolver=2 does not leak the extra feature into `lancedb-python`. ## Measurement Paired `maturin build --release --strip --target aarch64-apple-darwin --features fp16kernels` wheels. Source was `3be29228` plus this Cargo.toml change, which is this PR's tree (`878b2ae5` on `3be29228`). Same toolchain, profile, and packaging flags; only `rest-adapter` moved. | Artifact | Before | After | Delta | | --- | ---: | ---: | ---: | | Compressed wheel | 64,685,664 | 63,824,281 | −861,383 (−1.33%) | | `_lancedb.abi3.so` uncompressed | 148,255,440 | 146,275,936 | −1,979,504 (−1.34%) | This is macOS arm64, not Windows. It does not resolve the Windows wheel upload limit. Tonic still pulls Axum 0.8; the change only removes the adapter's Axum 0.7 stack from the production graph. --- rust/lancedb/Cargo.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index ebec41318..94857a124 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -106,6 +106,9 @@ sha2 = "0.10" [dev-dependencies] anyhow = "1" lance-testing = { workspace = true } +# RestAdapter is only used by cfg(test) code. Keep rest-adapter off the +# production remote feature so Python wheels do not resolve its Axum 0.7 stack. +lance-namespace-impls = { workspace = true, features = ["rest-adapter"] } tempfile = { workspace = true } random_word = { version = "0.4.3", features = ["en"] } tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] } @@ -169,7 +172,6 @@ remote = [ "dep:webbrowser", "dep:fs4", "lance-namespace-impls/rest", - "lance-namespace-impls/rest-adapter", ] # Publish LanceDB's internal metrics (currently object store request counts, # bytes, latency, errors, and throttles) through the `metrics` crate facade, From 254688df9633a9c49549303092ca048e4442182e Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Tue, 22 Sep 2026 08:03:27 -0700 Subject: [PATCH 10/15] feat: group a materialized view by GROUP BY (#4222) A view could only map source rows one to one, or one to many through a FROM-position item, so nothing could see all the rows sharing a key. A view query now takes `GROUP BY expr, ...` with aggregate projections, planned and executed by DataFusion over the lance scan. A group spans fragments, so a grouped view is recomputed in full whenever its source changes; each row's provenance is its group's smallest source row id. Such a query is stored as definition format 2, so a reader that predates grouping reports the view as unrefreshable instead of failing to parse it. Stacked on #4190. --- .../interfaces/MaterializedViewDefinition.md | 4 +- nodejs/__test__/materialized_view.test.ts | 2 +- nodejs/lancedb/materialized_view.ts | 11 +- python/python/lancedb/materialized_view.py | 9 +- .../python/tests/test_materialized_views.py | 2 +- rust/lancedb/src/materialized_view.rs | 148 +++++++--- rust/lancedb/src/materialized_view/grouped.rs | 261 ++++++++++++++++++ rust/lancedb/src/materialized_view/query.rs | 71 ++++- rust/lancedb/src/materialized_view/refresh.rs | 225 +++++++++++++-- rust/lancedb/src/remote/table.rs | 1 + 10 files changed, 656 insertions(+), 78 deletions(-) create mode 100644 rust/lancedb/src/materialized_view/grouped.rs diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md index a1ffb6e2b..68bcb1415 100644 --- a/docs/src/js/interfaces/MaterializedViewDefinition.md +++ b/docs/src/js/interfaces/MaterializedViewDefinition.md @@ -8,8 +8,8 @@ The query that defines a materialized view, as stored: `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] -[WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row -per element it returns. +[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM` +position yields one row per element it returns. ## Properties diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts index 352628208..5e5c08745 100644 --- a/nodejs/__test__/materialized_view.test.ts +++ b/nodejs/__test__/materialized_view.test.ts @@ -54,7 +54,7 @@ describe("materialized views", () => { // A newer writer's layout is reported, never guessed at. for (const newer of [ - `{"format":2,"query":${JSON.stringify(query)}}`, + `{"format":3,"query":${JSON.stringify(query)}}`, '{"kind":"select_v3","source_table":"people"}', ]) { expect(() => read(newer)).toThrow(/cannot refresh/); diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts index 15ae52274..51f8cfead 100644 --- a/nodejs/lancedb/materialized_view.ts +++ b/nodejs/lancedb/materialized_view.ts @@ -7,14 +7,17 @@ import { Table } from "./table"; /** Schema metadata key holding a materialized view's definition. */ export const DEFINITION_META_KEY = "mv.definition"; -/** The stored layout this version reads: `{"format": 1, "query": ""}`. */ -export const DEFINITION_FORMAT = 1; +/** + * The newest stored layout this version reads: `{"format": N, "query": ""}`, + * format 2 being a query with `GROUP BY`. + */ +export const DEFINITION_FORMAT = 2; /** * The query that defines a materialized view, as stored: * `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] - * [WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row - * per element it returns. + * [WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM` + * position yields one row per element it returns. */ export interface MaterializedViewDefinition { /** The defining query, in the canonical spelling the server stores. */ diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py index c487eb044..a836210cb 100644 --- a/python/python/lancedb/materialized_view.py +++ b/python/python/lancedb/materialized_view.py @@ -29,9 +29,10 @@ SelectArg = Union[ ] -DEFINITION_FORMAT = 1 -"""The stored layout this version reads: ``{"format": 1, "query": ""}``. -A ``kind`` key beside it is for readers older than the format number.""" +DEFINITION_FORMAT = 2 +"""The newest stored layout this version reads: ``{"format": N, "query": ""}``, +format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for +readers older than the format number.""" @dataclass @@ -40,7 +41,7 @@ class MaterializedViewDefinition: SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] - [WHERE predicate] [LIMIT n] + [WHERE predicate] [GROUP BY expr, ...] [LIMIT n] A Function in ``FROM`` position yields one row per element it returns. """ diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index cfe2405a4..c030ac1df 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -522,7 +522,7 @@ def test_stored_queries_and_legacy_layouts_are_read(): # A newer writer's layout is reported, never guessed at. for newer in ( - {"format": 2, "query": query}, + {"format": 3, "query": query}, {"kind": "select_v3", "source_table": "people"}, ): with pytest.raises(NotImplementedError, match="cannot refresh"): diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 74d831c82..067bde13d 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,6 +9,7 @@ //! query this version cannot maintain reads back as unrefreshable, not as a //! plain table. Queries, indexes and search work on the view unchanged. +mod grouped; mod query; pub mod refresh; @@ -79,13 +80,21 @@ const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions"; /// produces, which is what lets a query embed its own text. const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions"; -/// The layout this version writes under [`DEFINITION_META_KEY`]: -/// `{"format": 1, "query": ""}`, the query as -/// [`MaterializedViewDefinition::to_sql`] renders it. A reader refuses a -/// newer format rather than guess at it. The layout also carries -/// `"kind": "query"`, which readers older than the format number report -/// as an unrefreshable view instead of failing to read the metadata. -pub const DEFINITION_FORMAT: u64 = 1; +/// The newest layout this version reads under [`DEFINITION_META_KEY`]: +/// `{"format": N, "query": ""}`, the query as +/// [`MaterializedViewDefinition::to_sql`] renders it. Format 2 is a query +/// with `GROUP BY`; every other query is written as format 1, so a reader +/// that predates grouping reports a grouped view as unrefreshable rather +/// than failing to parse it. A reader refuses a newer format rather than +/// guess at it. The layout also carries `"kind": "query"`, which readers +/// older than the format number report as an unrefreshable view instead of +/// failing to read the metadata. +pub const DEFINITION_FORMAT: u64 = 2; + +/// The format `definition` is written in; see [`DEFINITION_FORMAT`]. +fn format_of(definition: &MaterializedViewDefinition) -> u64 { + if definition.is_grouped() { 2 } else { 1 } +} /// Legacy `kind` tag of the structured layout written before /// [`DEFINITION_FORMAT`] existed; still read, never written. @@ -240,6 +249,9 @@ pub struct MaterializedViewDefinition { pub projections: Vec, /// SQL predicate selecting the rows the view holds. pub filter: Option, + /// `GROUP BY` expressions. A grouped view holds one row per group and is + /// recomputed in full whenever its source changes. + pub group_by: Vec, /// Cap on the number of rows the view holds, in materialization order. pub limit: Option, } @@ -250,7 +262,7 @@ impl MaterializedViewDefinition { /// ```sql /// SELECT , ... /// FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] - /// [WHERE predicate] [LIMIT n] + /// [WHERE predicate] [GROUP BY expr, ...] [LIMIT n] /// ``` /// /// A Function in `FROM` position yields one row per element it returns; @@ -283,6 +295,11 @@ impl MaterializedViewDefinition { definition_to_metadata(self) } + /// Whether the query has a `GROUP BY`. + pub fn is_grouped(&self) -> bool { + !self.group_by.is_empty() + } + /// Whether the query is `SELECT *`. pub fn selects_star(&self) -> bool { matches!(self.projections.as_slice(), [p] if *p == ViewProjection::star()) @@ -359,7 +376,7 @@ struct LegacyDefinition { pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result { Ok(serde_json::json!({ "kind": "query", - "format": DEFINITION_FORMAT, + "format": format_of(definition), "query": definition.to_sql(), }) .to_string()) @@ -389,6 +406,13 @@ pub fn read_definition(metadata: &HashMap) -> Result) -> Result err, })?; + if definition.is_grouped() { + return grouped::plan(source_schema, definition, filter); + } // Projections are typed against the source, or for an unnested view // against the source with the list column replaced by its element // under the alias, where `c.chunk` is an ordinary nested path. @@ -616,34 +644,7 @@ pub(crate) fn plan( } if let Some(filter) = filter.as_deref() { - let expr = planner - .parse_filter(filter) - .map_err(|e| Error::InvalidInput { - message: format!("invalid view filter: {e}"), - })?; - ensure_immutable(&expr, |message| Error::InvalidInput { - message: format!("invalid view filter: {message}"), - })?; - let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput { - message: format!("invalid view filter: {message}"), - })?; - // A committed filter has to be usable as a predicate. - let read_schema = project_schema(&source_schema, &filter_inputs); - let data_type = Planner::new(read_schema.clone()) - .create_physical_expr(&expr) - .map_err(|e| Error::InvalidInput { - message: format!("invalid view filter: {e}"), - })? - .data_type(read_schema.as_ref()) - .map_err(|e| Error::InvalidInput { - message: format!("invalid view filter: {e}"), - })?; - if data_type != DataType::Boolean { - return Err(Error::InvalidInput { - message: format!("view filter must be a boolean predicate, not {data_type}"), - }); - } - inputs.extend(filter_inputs); + inputs.extend(plan_filter(&source_schema, filter)?); } let definition = MaterializedViewDefinition { @@ -655,6 +656,7 @@ pub(crate) fn plan( .map(|(output, expression)| ViewProjection { output, expression }) .collect(), filter, + group_by: Vec::new(), limit, }; let mut inputs: Vec = inputs @@ -671,6 +673,39 @@ pub(crate) fn plan( }) } +/// Check that `filter` is an immutable boolean predicate over `source_schema`, +/// returning the columns it reads. +pub(crate) fn plan_filter(source_schema: &SchemaRef, filter: &str) -> Result> { + let expr = Planner::new(source_schema.clone()) + .parse_filter(filter) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + ensure_immutable(&expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + let filter_inputs = resolve_inputs(source_schema, &expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + // A committed filter has to be usable as a predicate. + let read_schema = project_schema(source_schema, &filter_inputs); + let data_type = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })? + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + if data_type != DataType::Boolean { + return Err(Error::InvalidInput { + message: format!("view filter must be a boolean predicate, not {data_type}"), + }); + } + Ok(filter_inputs) +} + /// The schema a projection over an unnested view is planned against: the /// source's, with the list column replaced by its element type under the /// alias. @@ -1046,6 +1081,15 @@ impl PreparedDeclaration { /// read: the column the view projects it to, if any, otherwise an /// internal projection added here, named by [`input_column_name`]. pub fn input_column(&mut self, source_column: &str) -> Result { + // A grouped view's rows are groups; no source row carries a value into one. + if self.definition.is_grouped() { + return Err(Error::InvalidInput { + message: format!( + "a computed column on a grouped view cannot read source column \ + '{source_column}'; declare it on a view over the grouped view" + ), + }); + } if let Some(output) = self.lineage.get(source_column).and_then(|o| o.first()) { return Ok(output.clone()); } @@ -1357,6 +1401,7 @@ pub async fn prepare_declaration( .collect(), }, filter: filter.map(str::to_string), + group_by: Vec::new(), limit, }; prepare_definition(source, definition).await @@ -1383,6 +1428,26 @@ pub async fn prepare_declaration( /// # Ok(()) /// # } /// ``` +/// +/// A grouped view holds one row per group and is recomputed in full on +/// every refresh after a source change: +/// +/// ``` +/// # #![recursion_limit = "256"] +/// use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition}; +/// +/// # async fn declare(events: &lancedb::Table) -> Result<(), Box> { +/// let definition = MaterializedViewDefinition::from_sql( +/// "SELECT kind, count(*) AS n, array_agg(id) AS ids FROM events GROUP BY kind", +/// )?; +/// let view = prepare_definition(events, definition) +/// .await? +/// .create("events_by_kind") +/// .await?; +/// view.refresh().execute().await?; +/// # Ok(()) +/// # } +/// ``` pub async fn prepare_definition( source: &Table, definition: MaterializedViewDefinition, @@ -2067,6 +2132,7 @@ mod tests { }, ], filter: Some("age >= 18".into()), + group_by: Vec::new(), limit: Some(10), lateral: None, } @@ -3110,6 +3176,7 @@ mod tests { expression: "name".to_string(), }], filter: None, + group_by: Vec::new(), limit: None, } } @@ -3187,8 +3254,8 @@ mod tests { #[test] fn a_newer_format_is_reported_not_guessed() { assert_eq!( - read(r#"{"format":2,"query":"SELECT name FROM people"}"#).unwrap(), - Some(StoredDefinition::Newer { format: "2".into() }) + read(r#"{"format":3,"query":"SELECT name FROM people"}"#).unwrap(), + Some(StoredDefinition::Newer { format: "3".into() }) ); assert_eq!( read(r#"{"kind":"join"}"#).unwrap(), @@ -3201,6 +3268,7 @@ mod tests { r#"{"format":"one"}"#, r#"{"format":1}"#, r#"{"format":1,"query":"SELECT name FROM people GROUP BY name"}"#, + r#"{"format":2,"query":"SELECT name FROM people ORDER BY name"}"#, ] { assert!(read(stored).is_err(), "{stored}"); } @@ -3399,7 +3467,7 @@ mod tests { // The stored definition is the query alone; bindings live beside it. let stored: serde_json::Value = serde_json::from_str(&schema.metadata()[DEFINITION_META_KEY]).unwrap(); - assert_eq!(stored["format"], DEFINITION_FORMAT); + assert_eq!(stored["format"], 1); assert_eq!(view.definition().projections.len(), 2); assert_eq!(view.table().count_rows(None).await.unwrap(), 0); assert_eq!(conn.open_materialized_view("v").await.unwrap().name(), "v"); diff --git a/rust/lancedb/src/materialized_view/grouped.rs b/rust/lancedb/src/materialized_view/grouped.rs new file mode 100644 index 000000000..fc0dca430 --- /dev/null +++ b/rust/lancedb/src/materialized_view/grouped.rs @@ -0,0 +1,261 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Views with `GROUP BY`: one row per group, planned and executed by +//! DataFusion, and recomputed in full whenever the source changes, since a +//! group spans fragments. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef}; +use datafusion::catalog::default_table_source::provider_as_source; +use datafusion::datasource::empty::EmptyTable; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::context::SessionState; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::prelude::SessionContext; +use datafusion_common::TableReference; +use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion_expr::planner::{ContextProvider, ExprPlanner}; +use datafusion_expr::{ + AggregateUDF, HigherOrderUDF, LogicalPlan, ScalarUDF, TableSource, WindowUDF, +}; +use datafusion_sql::planner::SqlToRel; +use datafusion_sql::sqlparser::dialect::GenericDialect; +use datafusion_sql::sqlparser::parser::Parser; +use futures::StreamExt; +use lance::Dataset; +use lance_core::ROW_ID; +use lance_datafusion::exec::SessionContextExt; + +use super::refresh::to_view_batch; +use super::{ + MaterializedViewDefinition, Planned, SOURCE_ROW_ID_COLUMN, ViewProjection, ensure_immutable, + plan_filter, query, without_declarations, +}; +use crate::{Error, Result}; + +/// The name the source is planned under; never visible to the user. +const SOURCE: &str = "__source"; + +fn session() -> SessionState { + SessionStateBuilder::new().with_default_features().build() +} + +/// The query DataFusion runs: the view's projections plus the group's +/// smallest source row id, which stands as the row's provenance. The filter +/// is not here; the lance scan applies it, as for any other view. +fn sql(definition: &MaterializedViewDefinition) -> String { + let items: Vec = definition + .projections + .iter() + .map(|p| format!("{} AS {}", p.expression, query::ident_sql(&p.output))) + .collect(); + format!( + "SELECT {}, min({ROW_ID}) AS {ROW_ID} FROM {SOURCE} GROUP BY {}", + items.join(", "), + definition.group_by.join(", ") + ) +} + +struct Provider<'a> { + state: &'a SessionState, + source: Arc, +} + +impl ContextProvider for Provider<'_> { + fn get_table_source( + &self, + name: TableReference, + ) -> datafusion_common::Result> { + if name.schema().is_none() && name.table() == SOURCE { + Ok(self.source.clone()) + } else { + datafusion_common::plan_err!("a grouped view reads only its source, not '{name}'") + } + } + fn get_expr_planners(&self) -> &[Arc] { + self.state.expr_planners() + } + fn get_function_meta(&self, name: &str) -> Option> { + self.state.scalar_functions().get(name).cloned() + } + fn get_higher_order_meta(&self, name: &str) -> Option> { + self.state.higher_order_functions().get(name).cloned() + } + fn get_aggregate_meta(&self, name: &str) -> Option> { + self.state.aggregate_functions().get(name).cloned() + } + fn get_window_meta(&self, name: &str) -> Option> { + self.state.window_functions().get(name).cloned() + } + fn get_variable_type(&self, _: &[String]) -> Option { + None + } + fn options(&self) -> &ConfigOptions { + self.state.config_options() + } + fn udf_names(&self) -> Vec { + self.state.scalar_functions().keys().cloned().collect() + } + fn higher_order_function_names(&self) -> Vec { + self.state + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn udaf_names(&self) -> Vec { + self.state.aggregate_functions().keys().cloned().collect() + } + fn udwf_names(&self) -> Vec { + self.state.window_functions().keys().cloned().collect() + } +} + +fn logical_plan( + state: &SessionState, + source: Arc, + definition: &MaterializedViewDefinition, +) -> Result { + let invalid = |e: &dyn std::fmt::Display| Error::InvalidInput { + message: format!("invalid grouped view: {e}"), + }; + let statement = Parser::parse_sql(&GenericDialect {}, &sql(definition)) + .map_err(|e| invalid(&e))? + .pop() + .ok_or_else(|| invalid(&"empty query"))?; + SqlToRel::new(&Provider { state, source }) + .sql_statement_to_plan(statement) + .map_err(|e| invalid(&e)) +} + +/// Plan a grouped definition against the source schema. `filter` is the +/// canonical filter, already spelled the way [`super::plan`] stores it. +pub(super) fn plan( + source_schema: SchemaRef, + definition: &MaterializedViewDefinition, + filter: Option, +) -> Result { + query::check_grouping(definition)?; + let mut inputs = match &filter { + Some(filter) => plan_filter(&source_schema, filter)?, + None => Vec::new(), + }; + let mut projections: Vec = Vec::with_capacity(definition.projections.len()); + for p in &definition.projections { + if p.output == SOURCE_ROW_ID_COLUMN || p.output == ROW_ID { + return Err(Error::InvalidInput { + message: format!("view column name '{}' is reserved", p.output), + }); + } + if projections.iter().any(|seen| seen.output == p.output) { + return Err(Error::ColumnAlreadyExists { + name: p.output.clone(), + }); + } + let expression = + query::canonical_expr(&p.expression).map_err(|e| Error::InvalidExpression { + column: p.output.clone(), + message: e.to_string(), + })?; + projections.push(ViewProjection { + output: p.output.clone(), + expression, + }); + } + let group_by = definition + .group_by + .iter() + .map(|key| query::canonical_expr(key)) + .collect::>>()?; + let definition = MaterializedViewDefinition { + projections, + filter, + group_by, + ..definition.clone() + }; + + let mut fields = source_schema.fields().to_vec(); + fields.push(Arc::new(ArrowField::new(ROW_ID, DataType::UInt64, false))); + let empty = provider_as_source(Arc::new(EmptyTable::new(Arc::new(ArrowSchema::new( + fields, + ))))); + let planned = logical_plan(&session(), empty, &definition)?; + + let mut exprs = Vec::new(); + planned + .apply(|node| { + exprs.extend(node.expressions()); + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| Error::InvalidInput { + message: format!("invalid grouped view: {e}"), + })?; + for expr in &exprs { + ensure_immutable(expr, |message| Error::InvalidInput { + message: format!("invalid grouped view: {message}"), + })?; + inputs.extend( + expr.column_refs() + .into_iter() + .map(|c| c.name.clone()) + .filter(|name| name != ROW_ID && source_schema.field_with_name(name).is_ok()), + ); + } + inputs.sort(); + inputs.dedup(); + + let fields = planned + .schema() + .fields() + .iter() + .filter(|f| f.name() != ROW_ID) + .map(|f| without_declarations(f)) + .collect(); + Ok(Planned { + definition, + fields, + lineage: HashMap::new(), + inputs, + }) +} + +/// Every group of `source`, as view rows in `schema`. +pub(super) async fn stream( + source: &Dataset, + definition: &MaterializedViewDefinition, + inputs: &[String], + schema: SchemaRef, + rows_written: Arc, +) -> Result { + let mut scanner = source.scan(); + scanner.with_row_id(); + if let Some(filter) = &definition.filter { + scanner.filter(filter)?; + } + scanner.project(inputs)?; + let scan: SendableRecordBatchStream = scanner.try_into_stream().await?.into(); + + let state = session(); + let ctx = SessionContext::new_with_state(state.clone()); + let source = ctx.read_one_shot(scan)?.into_view(); + let planned = logical_plan(&state, provider_as_source(source), definition)?; + let groups = ctx + .execute_logical_plan(planned) + .await? + .execute_stream() + .await?; + + let out_schema = schema.clone(); + let mapped = groups.map(move |batch| { + let batch = to_view_batch(&batch?, &out_schema)?; + rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(batch) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) +} diff --git a/rust/lancedb/src/materialized_view/query.rs b/rust/lancedb/src/materialized_view/query.rs index 35af77f0b..c1a987b2c 100644 --- a/rust/lancedb/src/materialized_view/query.rs +++ b/rust/lancedb/src/materialized_view/query.rs @@ -18,8 +18,9 @@ //! query: fail closed, never materialize it at the wrong cardinality. use datafusion_sql::sqlparser::ast::{ - Expr, FunctionArg, FunctionArgExpr, JoinOperator, LimitClause, ObjectName, ObjectNamePart, - Query, SelectItem, SetExpr, Statement, TableFactor, TableFunctionArgs, TableWithJoins, Value, + Expr, FunctionArg, FunctionArgExpr, GroupByExpr, JoinOperator, LimitClause, ObjectName, + ObjectNamePart, Query, SelectItem, SetExpr, Statement, TableFactor, TableFunctionArgs, + TableWithJoins, Value, }; use datafusion_sql::sqlparser::dialect::GenericDialect; use datafusion_sql::sqlparser::keywords::{ @@ -40,7 +41,8 @@ fn invalid(message: impl Into) -> Error { } const SHAPE: &str = "a materialized view is defined by `SELECT columns FROM table \ - [, function(args) AS alias | , UNNEST(column) AS alias] [WHERE predicate] [LIMIT n]`"; + [, function(args) AS alias | , UNNEST(column) AS alias] [WHERE predicate] \ + [GROUP BY expr, ...] [LIMIT n]`"; /// Whether `name` must be delimited to read back as this identifier: bare, /// the parser would take it as a keyword, or a different spelling. @@ -394,14 +396,59 @@ fn extract(query: &Query) -> Result { Some(_) => return Err(invalid("view limit must be a plain `LIMIT n`")), }; - Ok(MaterializedViewDefinition { + if select.having.is_some() { + return Err(invalid("HAVING is not supported in a view query")); + } + let group_by = match &select.group_by { + GroupByExpr::Expressions(exprs, modifiers) + if modifiers.is_empty() + && !exprs.iter().any(|e| { + matches!(e, Expr::Rollup(_) | Expr::Cube(_) | Expr::GroupingSets(_)) + }) => + { + exprs.iter().map(|e| e.to_string()).collect() + } + _ => { + return Err(invalid( + "a view groups by a plain `GROUP BY expr, ...` list", + )); + } + }; + + let definition = MaterializedViewDefinition { source_table, source_namespace, lateral, projections, filter: select.selection.as_ref().map(|e| e.to_string()), + group_by, limit, - }) + }; + check_grouping(&definition)?; + Ok(definition) +} + +/// The clauses a grouped view cannot combine with: each would have to be +/// maintained per group rather than per source row. +pub fn check_grouping(definition: &MaterializedViewDefinition) -> Result<()> { + if !definition.is_grouped() { + return Ok(()); + } + if definition.lateral.is_some() { + return Err(invalid( + "GROUP BY cannot be combined with a FROM-position item; group in one view \ + and expand in a view over it", + )); + } + if definition.selects_star() { + return Err(invalid( + "a grouped view selects its keys and aggregates, not `*`", + )); + } + if definition.limit.is_some() { + return Err(invalid("LIMIT is not supported together with GROUP BY")); + } + Ok(()) } /// The canonical query for `definition`; [`parse`] reads it back equal. @@ -451,6 +498,9 @@ pub fn render(definition: &MaterializedViewDefinition) -> String { if let Some(filter) = &definition.filter { sql.push_str(&format!(" WHERE {filter}")); } + if definition.is_grouped() { + sql.push_str(&format!(" GROUP BY {}", definition.group_by.join(", "))); + } if let Some(limit) = definition.limit { sql.push_str(&format!(" LIMIT {limit}")); } @@ -489,6 +539,10 @@ mod tests { "SELECT id, c.text FROM docs, chunk(upper(body)) AS c", ), ("SELECT * FROM `select`.t", "SELECT * FROM `select`.t"), + ( + "select k, array_agg(id) as ids, count(*) n from t where x > 1 group by k", + "SELECT k, array_agg(id) AS ids, count(*) AS n FROM t WHERE x > 1 GROUP BY k", + ), ( "SELECT meta.title AS title, id AS id FROM t", "SELECT meta.title, id FROM t", @@ -547,7 +601,6 @@ mod tests { #[test] fn unsupported_clauses_are_refused() { for sql in [ - "SELECT id FROM t GROUP BY id", "SELECT id FROM t ORDER BY id", "SELECT DISTINCT id FROM t", "SELECT id FROM t LIMIT 5 OFFSET 2", @@ -567,6 +620,12 @@ mod tests { "SELECT *, id FROM t", "WITH q AS (SELECT 1) SELECT id FROM t", "SELECT id FROM t HAVING id > 1", + "SELECT k, count(*) AS n FROM t GROUP BY k HAVING count(*) > 1", + "SELECT k FROM t GROUP BY ALL", + "SELECT k FROM t GROUP BY ROLLUP (k)", + "SELECT * FROM t GROUP BY k", + "SELECT k FROM t GROUP BY k LIMIT 5", + "SELECT k, c.x FROM t, UNNEST(xs) AS c GROUP BY k", ] { assert!(parse(sql).is_err(), "{sql}"); } diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 0fc9b99c5..39e1d8b9c 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -297,6 +297,22 @@ pub(crate) async fn execute_refresh( }); } + // A group spans fragments: there is no per-fragment increment. + if definition.is_grouped() { + return rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + &inputs, + None, + persist.is_some(), + expected_incarnation, + ) + .await; + } let watermark = watermark.filter(|_| view_intact); match plan_increment( &source_ds, @@ -629,9 +645,17 @@ fn same_meaning( || stored.limit != replanned.limit || stored.projections.len() != replanned.projections.len() || stored.filter.is_some() != replanned.filter.is_some() + || stored.group_by.len() != replanned.group_by.len() { return false; } + // Lance's planner does not read aggregates; a grouped definition is + // compared in its canonical spelling, which planning produced. + if stored.is_grouped() { + return stored.projections == replanned.projections + && stored.group_by == replanned.group_by + && stored.filter == replanned.filter; + } let schema = match unnest { None => source_schema.clone(), Some(unnest) => match super::flattened_schema(source_schema, unnest) { @@ -1288,6 +1312,9 @@ async fn compute_stream( schema: SchemaRef, rows_written: Arc, ) -> Result { + if definition.is_grouped() { + return super::grouped::stream(source, definition, inputs, schema, rows_written).await; + } let RowScope { fragments, created_after, @@ -1363,31 +1390,42 @@ async fn compute_stream( None => batch, Some(expanded) => expanded.apply(&batch)?, }; - let mut columns = Vec::with_capacity(out_schema.fields().len()); - for field in out_schema.fields() { - if computed_column_from_field(field).is_some() { - columns.push(new_null_array(field.data_type(), batch.num_rows())); - continue; - } - let name = if field.name() == SOURCE_ROW_ID_COLUMN { - ROW_ID - } else { - field.name() - }; - let column = batch.column_by_name(name).ok_or_else(|| { - DataFusionError::Internal(format!( - "view column '{}' is not produced by the view's definition", - field.name() - )) - })?; - columns.push(column.clone()); - } + let batch = to_view_batch(&batch, &out_schema)?; rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); - Ok(RecordBatch::try_new(out_schema.clone(), columns)?) + Ok(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) } +/// `batch` in the view's physical schema: the row id becomes the +/// provenance column and computed columns are written NULL for their owner +/// to fill. +pub(super) fn to_view_batch( + batch: &RecordBatch, + out_schema: &SchemaRef, +) -> datafusion::common::Result { + let mut columns = Vec::with_capacity(out_schema.fields().len()); + for field in out_schema.fields() { + if computed_column_from_field(field).is_some() { + columns.push(new_null_array(field.data_type(), batch.num_rows())); + continue; + } + let name = if field.name() == SOURCE_ROW_ID_COLUMN { + ROW_ID + } else { + field.name() + }; + let column = batch.column_by_name(name).ok_or_else(|| { + DataFusionError::Internal(format!( + "view column '{}' is not produced by the view's definition", + field.name() + )) + })?; + columns.push(column.clone()); + } + Ok(RecordBatch::try_new(out_schema.clone(), columns)?) +} + /// The post-scan half of an unnested view's refresh: the scan reads /// `raw_inputs` plus the row id, and each batch is unnested on the list /// column, filtered, then projected by expressions typed against @@ -3641,6 +3679,7 @@ mod tests { }, ], filter: None, + group_by: Vec::new(), limit: None, lateral: None, }; @@ -3675,6 +3714,7 @@ mod tests { expression: "x".into(), }], filter: None, + group_by: Vec::new(), limit: None, lateral: None, }; @@ -4304,4 +4344,149 @@ mod tests { 3 ); } + + async fn grouped_source(conn: &Connection) -> Table { + conn.create_table( + "src", + record_batch!(("k", Int32, [1, 1, 2, 3]), ("x", Int32, [10, 11, 20, 30])).unwrap(), + ) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap() + } + + async fn declare(source: &Table, name: &str, sql: &str) -> Result { + crate::materialized_view::prepare_definition( + source, + MaterializedViewDefinition::from_sql(sql)?, + ) + .await? + .create(name) + .await + } + + /// Rows of `table` as `columns` rendered to text, sorted. + async fn rows(table: &Table, columns: &[&str]) -> Vec { + let batches = table + .query() + .select(Select::columns(columns)) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut rows = Vec::new(); + for batch in &batches { + let formatters: Vec<_> = batch + .columns() + .iter() + .map(|c| { + arrow_cast::display::ArrayFormatter::try_new(c.as_ref(), &Default::default()) + .unwrap() + }) + .collect(); + for row in 0..batch.num_rows() { + let cells: Vec = formatters + .iter() + .map(|f| f.value(row).to_string()) + .collect(); + rows.push(cells.join(" ")); + } + } + rows.sort(); + rows + } + + #[tokio::test] + async fn a_grouped_view_holds_one_row_per_group() { + let conn = connect("memory://").execute().await.unwrap(); + let source = grouped_source(&conn).await; + let view = declare( + &source, + "groups", + "SELECT k, array_agg(x) AS xs, sum(x) AS total FROM src WHERE x < 30 GROUP BY k", + ) + .await + .unwrap(); + assert_eq!( + view.definition().to_sql(), + "SELECT k, array_agg(x) AS xs, sum(x) AS total FROM src WHERE x < 30 GROUP BY k" + ); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 2); + assert_eq!(rows(view.table(), &["k", "total"]).await, ["1 21", "2 20"]); + let unchanged = view.refresh().execute().await.unwrap(); + assert_eq!(unchanged.mode, RefreshMode::NoOp); + + // Any source change recomputes every group. + source + .add(record_batch!(("k", Int32, [2]), ("x", Int32, [5])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(rows(view.table(), &["k", "total"]).await, ["1 21", "2 25"]); + + // A view over the groups expands them again. + let expanded = declare( + view.table(), + "elements", + "SELECT k, e AS x FROM groups, UNNEST(xs) AS e", + ) + .await + .unwrap(); + expanded.refresh().execute().await.unwrap(); + assert_eq!( + rows(expanded.table(), &["k", "x"]).await, + ["1 10", "1 11", "2 20", "2 5"] + ); + } + + /// Provenance is one source row per group, so the view's row-id keys + /// stay unique. + #[tokio::test] + async fn a_grouped_view_records_one_source_row_per_group() { + let conn = connect("memory://").execute().await.unwrap(); + let source = grouped_source(&conn).await; + let view = declare( + &source, + "groups", + "SELECT k, count(*) AS n FROM src GROUP BY k", + ) + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + let ids = rows(view.table(), &[SOURCE_ROW_ID_COLUMN]).await; + assert_eq!(ids.len(), 3); + assert_eq!(ids.iter().collect::>().len(), 3, "{ids:?}"); + } + + #[tokio::test] + async fn a_grouped_view_is_planned_at_declaration() { + let conn = connect("memory://").execute().await.unwrap(); + let source = grouped_source(&conn).await; + for sql in [ + // x is neither a key nor aggregated + "SELECT k, x FROM src GROUP BY k", + "SELECT k, random() AS r FROM src GROUP BY k", + "SELECT k, count(*) AS n FROM src GROUP BY missing", + "SELECT k, count(*) AS __source_row_id FROM src GROUP BY k", + "SELECT k, count(*) AS k FROM src GROUP BY k", + ] { + assert!(declare(&source, "v", sql).await.is_err(), "{sql}"); + } + let mut grouped = crate::materialized_view::prepare_definition( + &source, + MaterializedViewDefinition::from_sql("SELECT k, count(*) AS n FROM src GROUP BY k") + .unwrap(), + ) + .await + .unwrap(); + assert!(grouped.input_column("x").is_err()); + } } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 0c51f50ae..6bbb49adf 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2144,6 +2144,7 @@ impl BaseTable for RemoteTable { }) .collect(), filter: response.filter, + group_by: Vec::new(), limit: response.limit, }, }; From 814de30c5a16416382f4affdc3b2ae8ade46369f Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 22 Sep 2026 09:54:23 -0700 Subject: [PATCH 11/15] feat: return a cleanup job from drop_function (#4237) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping a Function can leave its content to a server-side cleanup job, so the name drop and the content deletion become separate events a caller may want to wait on. `drop_function_async` returns the unbind result alongside a `Job` for the cleanup, the same shape `drop_table_async` and `drop_materialized_view_async` already use: a `202` carries the job id, and a `200` — an inline deletion, or a name that was not bound — yields an already-finished job with no id. A `202` without a usable job id is rejected rather than silently reported as finished. `drop_function` keeps its `bool` result and now delegates, so nothing changes for callers that do not care when the content goes. Available on `Connection` in Rust and on both the sync and asyncio Python connections. --- python/python/lancedb/_lancedb.pyi | 3 ++ python/python/lancedb/db.py | 44 +++++++++++++--- python/python/lancedb/remote/db.py | 6 +++ python/src/connection.rs | 15 ++++++ rust/lancedb/src/connection.rs | 16 ++++++ rust/lancedb/src/database.rs | 12 +++++ rust/lancedb/src/remote/db.rs | 81 +++++++++++++++++++++++++++++- 7 files changed, 169 insertions(+), 8 deletions(-) diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 7c6611608..6d873850d 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -153,6 +153,9 @@ class Connection(object): async def get_function(self, name: str, version: str) -> str: ... async def list_functions(self) -> List[str]: ... async def drop_function(self, name: str, version: str) -> bool: ... + async def drop_function_async( + self, name: str, version: str + ) -> Tuple[bool, Job]: ... async def create_secret( self, name: str, value: str, namespace_path: Optional[List[str]] = None ) -> None: ... diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index d17e897cf..b0db22532 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -18,6 +18,7 @@ from typing import ( Literal, Optional, Sequence, + Tuple, Union, ) from uuid import UUID @@ -828,12 +829,26 @@ class DBConnection(EnforceOverrides): ) def drop_function(self, name: str, *, version: str) -> bool: - """Remove the current Function name binding from the remote catalog. + """Drop a Function name and the object it was bound to. - The requested version must exist in the currently named object. - Object history and existing computed-column references are retained. - Returns True when the name was removed and False when it was absent. - Local connections raise NotImplementedError. + The requested version must exist in the currently named object. Returns + True when the name was removed and False when it was absent. The + object's content is deleted with it, which may finish after this + returns; use :meth:`drop_function_async` to wait for that. Local + connections raise NotImplementedError. + """ + raise NotImplementedError( + "Function catalog operations are not supported for this connection type" + ) + + def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + """Drop a Function name and return its cleanup job. + + The name is unbound before this returns; the object's content may still + be being deleted. Call :meth:`Job.wait` to wait for that to finish. When + the server deletes inline, or when nothing was bound, the returned job + is already finished and has no id. Local connections raise + NotImplementedError. """ raise NotImplementedError( "Function catalog operations are not supported for this connection type" @@ -1683,6 +1698,11 @@ class LanceDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version)) + return dropped, Job(job) + @override def create_secret( self, name: str, value: str, *, namespace_path: Optional[List[str]] = None @@ -2596,9 +2616,21 @@ class AsyncConnection(object): ] async def drop_function(self, name: str, *, version: str) -> bool: - """Remove the current name binding, retaining the object and its history.""" + """Drop a Function name and the object it was bound to.""" return await self._inner.drop_function(name, version) + async def drop_function_async( + self, name: str, *, version: str + ) -> Tuple[bool, AsyncJob]: + """Drop a Function name and return its cleanup job. + + The name is unbound before this returns; the object's content may still + be being deleted. Await :meth:`AsyncJob.wait` to wait for that to + finish. + """ + dropped, job = await self._inner.drop_function_async(name, version) + return dropped, AsyncJob(job) + async def create_secret( self, name: str, value: str, *, namespace_path: Optional[List[str]] = None ) -> None: diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index c9e857495..ef40ef007 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -16,6 +16,7 @@ from typing import ( List, Optional, Sequence, + Tuple, Union, ) from urllib.parse import urlparse @@ -876,6 +877,11 @@ class RemoteDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version)) + return dropped, Job(job) + @override def create_secret( self, name: str, value: str, *, namespace_path: Optional[List[str]] = None diff --git a/python/src/connection.rs b/python/src/connection.rs index a972f4351..3385e6b2d 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -768,6 +768,21 @@ impl Connection { }) } + pub fn drop_function_async( + self_: PyRef<'_, Self>, + name: String, + version: String, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + inner + .drop_function_async(name, version) + .await + .infer_error() + .map(|(dropped, job)| (dropped, crate::job::Job::new(job))) + }) + } + #[pyo3(signature = (name, value, namespace_path=None))] pub fn create_secret( self_: PyRef<'_, Self>, diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 7234a911f..d43f8f744 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -649,6 +649,22 @@ impl Connection { .await } + /// Start dropping a Function and return its cleanup job. + /// + /// The name is unbound before this returns; the object's content may still be being + /// deleted. Await [`Job::wait`][crate::job::Job::wait] to wait for that to finish. When + /// the server deletes inline, or when nothing was bound, the returned job is already + /// finished and has no id. Local databases return [`Error::NotSupported`]. + pub async fn drop_function_async( + &self, + name: impl AsRef, + version: impl AsRef, + ) -> Result<(bool, crate::job::Job)> { + self.internal + .drop_function_async(name.as_ref(), version.as_ref()) + .await + } + /// Create a named Secret in this database. /// /// Fails if the name is taken, so a create can never silently become a diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 0adac778c..91b200043 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -393,6 +393,18 @@ pub trait Database: async fn drop_function(&self, _name: &str, _version: &str) -> Result { function_catalog_not_supported() } + /// Start dropping a Function and return a handle to the cleanup job. + /// + /// Backends without asynchronous cleanup complete the drop before returning an + /// already-finished job. + async fn drop_function_async( + &self, + name: &str, + version: &str, + ) -> Result<(bool, crate::job::Job)> { + let dropped = self.drop_function(name, version).await?; + Ok((dropped, crate::job::Job::new_done())) + } /// Create a named Secret in this database. Fails if the name is taken, so /// a create can never silently become a rotation. async fn create_secret( diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 7061ac979..7a59db665 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1005,6 +1005,10 @@ impl Database for RemoteDatabase { } async fn drop_function(&self, name: &str, version: &str) -> Result { + Ok(self.drop_function_async(name, version).await?.0) + } + + async fn drop_function_async(&self, name: &str, version: &str) -> Result<(bool, Job)> { let function_id = build_object_identifier("Function name", name, &[])?; let req = self .client @@ -1014,8 +1018,34 @@ impl Database for RemoteDatabase { })); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; - let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?; - Ok(response.dropped) + let status = response.status(); + let body = response.text().await.err_to_http(request_id.clone())?; + let dropped: RemoteDropFunctionResponse = + serde_json::from_str(&body).map_err(|source| Error::Http { + source: Box::new(source), + request_id: request_id.clone(), + status_code: Some(status), + })?; + let job = match status { + StatusCode::OK => Job::new_done(), + StatusCode::ACCEPTED => { + let job_id = extract_job_id(&body).ok_or_else(|| Error::Http { + source: "asynchronous Function drop response did not contain a valid job_id" + .into(), + request_id, + status_code: Some(status), + })?; + Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))) + } + _ => { + return Err(Error::Http { + source: "Function drop must return 200 OK or 202 Accepted".into(), + request_id, + status_code: Some(status), + }); + } + }; + Ok((dropped.dropped, job)) } async fn create_secret( @@ -1783,6 +1813,53 @@ mod tests { assert_eq!(job.id(), Some("j1-mv-drop")); } + #[tokio::test] + async fn test_drop_function_async_returns_job() { + let db = super::RemoteDatabase::new_mock(|request| { + assert_eq!(request.method(), "POST"); + assert_eq!(request.url().path(), "/v1/function/embed/drop"); + http::Response::builder() + .status(202) + .body(serde_json::json!({"dropped": true, "job_id": "j1-fn-drop"}).to_string()) + .unwrap() + }); + let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap(); + assert!(dropped); + assert_eq!(job.id(), Some("j1-fn-drop")); + } + + /// An unbound name and an inline deletion both answer `200`: the name is gone and + /// nothing is left to wait for, so the job is already finished. + #[tokio::test] + async fn test_drop_function_async_completed_inline() { + let db = super::RemoteDatabase::new_mock(|_| { + http::Response::builder() + .status(200) + .body(serde_json::json!({"dropped": false}).to_string()) + .unwrap() + }); + let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap(); + assert!(!dropped); + assert_eq!(job.id(), None); + assert_eq!(job.status().await.unwrap(), "finished"); + job.wait().await.unwrap(); + } + + #[tokio::test] + async fn test_drop_function_async_rejects_incomplete_acceptance() { + for body in [ + r#"{"dropped":true}"#, + r#"{"dropped":true,"job_id":""}"#, + r#"{"dropped":true,"job_id":null}"#, + ] { + let db = super::RemoteDatabase::new_mock(move |_| { + http::Response::builder().status(202).body(body).unwrap() + }); + let error = db.drop_function_async("embed", "1").await.err().unwrap(); + assert!(error.to_string().contains("valid job_id")); + } + } + #[tokio::test] async fn test_drop_materialized_view_completed_inline() { let db = super::RemoteDatabase::new_mock(|request| { From e020e137444c46bc561d15490ace000332acd475 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Tue, 22 Sep 2026 10:16:20 -0700 Subject: [PATCH 12/15] feat: group a materialized view by the IVF partition of a vector column (#4223) Grouping near neighbours together, so an all-pairs comparison runs per bucket instead of over the whole table, needs the partition an IVF index assigns each vector. `ivf_partition(column)` returns that partition, assigned by the centroids and distance type of the IVF index on the column. It is bound when the view is declared and again at every refresh; an index retrain commits a new source version, so the next refresh regroups. A column without an IVF index, or with two, is refused. Stacked on #4222. --- rust/lancedb/src/materialized_view.rs | 4 + rust/lancedb/src/materialized_view/grouped.rs | 286 ++++++++++++++++- rust/lancedb/src/materialized_view/refresh.rs | 298 ++++++++++++++++++ 3 files changed, 578 insertions(+), 10 deletions(-) diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 067bde13d..c4456dbe1 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -10,6 +10,7 @@ //! plain table. Queries, indexes and search work on the view unchanged. mod grouped; +pub use grouped::IVF_PARTITION; mod query; pub mod refresh; @@ -1597,6 +1598,9 @@ async fn prepare_with( lineage, .. } = plan(source_schema.clone(), &definition, staging.as_ref())?; + if definition.is_grouped() { + grouped::check(native.dataset.get().await?.as_ref(), &definition).await?; + } // What later projections (`input_column`) are planned against: for an // unnested view the flattened schema, where the alias is a column. let planning_schema = match physical_unnest(&definition, staging.as_ref())? { diff --git a/rust/lancedb/src/materialized_view/grouped.rs b/rust/lancedb/src/materialized_view/grouped.rs index fc0dca430..00e14527a 100644 --- a/rust/lancedb/src/materialized_view/grouped.rs +++ b/rust/lancedb/src/materialized_view/grouped.rs @@ -9,28 +9,40 @@ use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use arrow_array::cast::AsArray; +use arrow_array::{Array, FixedSizeListArray, UInt32Array}; +use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef}; use datafusion::catalog::default_table_source::provider_as_source; use datafusion::datasource::empty::EmptyTable; +use datafusion::execution::FunctionRegistry; use datafusion::execution::SessionStateBuilder; use datafusion::execution::context::SessionState; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::prelude::SessionContext; +use datafusion_common::DataFusionError; use datafusion_common::TableReference; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion_expr::planner::{ContextProvider, ExprPlanner}; use datafusion_expr::{ - AggregateUDF, HigherOrderUDF, LogicalPlan, ScalarUDF, TableSource, WindowUDF, + AggregateUDF, ColumnarValue, Expr, HigherOrderUDF, LogicalPlan, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, TableSource, Volatility, WindowUDF, }; use datafusion_sql::planner::SqlToRel; use datafusion_sql::sqlparser::dialect::GenericDialect; use datafusion_sql::sqlparser::parser::Parser; use futures::StreamExt; use lance::Dataset; +use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; use lance_core::ROW_ID; use lance_datafusion::exec::SessionContextExt; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::vector::ivf::{IvfTransformer, new_ivf_transformer}; +use lance_linalg::distance::DistanceType; +use lance_linalg::kernels::normalize_fsl; +use uuid::Uuid; use super::refresh::to_view_batch; use super::{ @@ -42,8 +54,262 @@ use crate::{Error, Result}; /// The name the source is planned under; never visible to the user. const SOURCE: &str = "__source"; -fn session() -> SessionState { - SessionStateBuilder::new().with_default_features().build() +/// `ivf_partition(column)`: the partition of the IVF index on `column` that +/// each vector falls in, assigned by that index's centroids and distance +/// type. Bound per refresh, so a retrained index regroups the view. +pub const IVF_PARTITION: &str = "ivf_partition"; + +/// The index `ivf_partition(column)` assigns by. +#[derive(Debug, Clone)] +struct IvfBinding { + index: Uuid, + transformer: Arc, + /// A cosine index assigns L2 over unit vectors, as lance's index path does. + normalize: bool, +} + +/// `bindings` maps a source column to its index; planning runs unbound. +#[derive(Debug)] +struct IvfPartition { + signature: Signature, + bindings: HashMap, +} + +impl PartialEq for IvfPartition { + fn eq(&self, other: &Self) -> bool { + self.bindings.len() == other.bindings.len() + && self.bindings.iter().all(|(column, binding)| { + other + .bindings + .get(column) + .is_some_and(|b| b.index == binding.index) + }) + } +} + +impl Eq for IvfPartition {} + +impl std::hash::Hash for IvfPartition { + fn hash(&self, state: &mut H) { + let mut bound: Vec<_> = self.bindings.iter().map(|(c, b)| (c, b.index)).collect(); + bound.sort(); + bound.hash(state); + } +} + +impl ScalarUDFImpl for IvfPartition { + fn name(&self) -> &str { + IVF_PARTITION + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> datafusion_common::Result { + // Float vectors under L2/dot/cosine, byte vectors under Hamming; the + // binding checks the index's metric against the element type. + match arg_types { + [DataType::FixedSizeList(element, _)] + if element.data_type().is_floating() || *element.data_type() == DataType::UInt8 => + { + Ok(DataType::UInt32) + } + other => datafusion_common::plan_err!( + "{IVF_PARTITION} takes one vector column, not {other:?}" + ), + } + } + + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> datafusion_common::Result { + // The argument is a bare column (checked at planning), so its field + // names the column the binding was made for. + let column = args.arg_fields[0].name(); + let Some(binding) = self.bindings.get(column) else { + return datafusion_common::exec_err!("{IVF_PARTITION}({column}) has no index bound"); + }; + let vectors = args.args[0].to_array(args.number_rows)?; + let vectors = vectors.as_fixed_size_list(); + let normalized; + let vectors = if binding.normalize { + normalized = + normalize_fsl(vectors).map_err(|e| DataFusionError::External(Box::new(e)))?; + &normalized + } else { + vectors + }; + let partitions = binding + .transformer + .compute_partitions(vectors) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + // A null vector falls in no partition, and neither does one the + // assigner could not place (a zero vector under cosine): a NULL bucket + // is honest, membership in bucket 0 is not. + let partitions = UInt32Array::new( + partitions.values().clone(), + NullBuffer::union(vectors.nulls(), partitions.nulls()), + ); + Ok(ColumnarValue::Array(Arc::new(partitions))) + } +} + +fn session(bindings: HashMap) -> SessionState { + let mut state = SessionStateBuilder::new().with_default_features().build(); + let ivf = IvfPartition { + signature: Signature::any(1, Volatility::Immutable), + bindings, + }; + state + .register_udf(Arc::new(ScalarUDF::new_from_impl(ivf))) + .expect("registering a scalar function cannot fail"); + state +} + +/// The columns `plan` calls `ivf_partition` on. The argument must be a bare +/// column: its field is how the bound function finds the column's index. +fn ivf_columns(plan: &LogicalPlan) -> Result> { + let mut columns = Vec::new(); + let mut misuse = None; + plan.apply(|node| { + for expr in node.expressions() { + expr.apply(|e| { + if let Expr::ScalarFunction(call) = e + && call.name() == IVF_PARTITION + { + match call.args.as_slice() { + [Expr::Column(column)] => columns.push(column.name.clone()), + _ => misuse = Some(e.to_string()), + } + } + Ok(TreeNodeRecursion::Continue) + })?; + } + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| Error::InvalidInput { + message: format!("invalid grouped view: {e}"), + })?; + if let Some(call) = misuse { + return Err(Error::InvalidInput { + message: format!("{IVF_PARTITION} takes a vector column of the source, not `{call}`"), + }); + } + columns.sort(); + columns.dedup(); + Ok(columns) +} + +/// Refuse a grouped `definition` whose functions cannot be bound over +/// `source`, such as `ivf_partition` on a column without an IVF index. +pub(super) async fn check(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> { + bind(source, definition).await.map(|_| ()) +} + +/// Bind every `ivf_partition(column)` in `definition` to the IVF index on +/// that column of `source`. +async fn bind( + source: &Dataset, + definition: &MaterializedViewDefinition, +) -> Result> { + let schema = Arc::new(ArrowSchema::from(source.schema())); + let planned = logical_plan(&session(HashMap::new()), empty_source(&schema), definition)?; + let mut bindings = HashMap::new(); + for column in ivf_columns(&planned)? { + bindings.insert(column.clone(), bind_column(source, &column).await?); + } + Ok(bindings) +} + +async fn bind_column(source: &Dataset, column: &str) -> Result { + let field = source + .schema() + .field(column) + .ok_or_else(|| Error::InvalidInput { + message: format!("{IVF_PARTITION}: the source has no column '{column}'"), + })?; + // One index, whose segments all assign by one model. A segment trained + // on its own fragments carries its own centroids, and grouping every row + // by one segment's model would bucket the other segments' rows wrongly. + let mut found: Option<(String, Uuid, DistanceType, FixedSizeListArray)> = None; + for index in source.load_indices().await?.iter() { + if index.fields != [field.id] { + continue; + } + let Ok(vector) = source + .open_vector_index(column, &index.uuid, &NoOpMetricsCollector) + .await + else { + continue; + }; + let Some(centroids) = vector.ivf_model().centroids.clone() else { + continue; + }; + let metric = vector.metric_type(); + match &found { + None => found = Some((index.name.clone(), index.uuid, metric, centroids)), + Some((name, _, seen_metric, seen)) if *name == index.name => { + if *seen_metric != metric || seen.to_data() != centroids.to_data() { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}): the segments of index '{name}' were \ + trained separately and assign by different models; rebuild \ + the index before grouping by it" + ), + }); + } + } + Some((name, ..)) => { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}) is ambiguous: indices '{name}' and '{}' \ + both cover it", + index.name + ), + }); + } + } + } + let byte_vectors = matches!( + field.data_type(), + DataType::FixedSizeList(element, _) if *element.data_type() == DataType::UInt8 + ); + let Some((_, uuid, metric, centroids)) = found else { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}) needs an IVF vector index on '{column}'; create one first" + ), + }); + }; + // The assigner pairs byte vectors with Hamming and float vectors with the + // rest; an index of the other kind cannot place this column's values. + if byte_vectors != (metric == DistanceType::Hamming) { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}): the index assigns by {metric}, which does not \ + apply to {}", + field.data_type() + ), + }); + } + // Lance's index path assigns cosine by L2 over unit vectors. + let normalize = metric == DistanceType::Cosine; + let distance = if normalize { DistanceType::L2 } else { metric }; + Ok(IvfBinding { + index: uuid, + transformer: Arc::new(new_ivf_transformer(centroids, distance, vec![])), + normalize, + }) +} + +fn empty_source(source_schema: &SchemaRef) -> Arc { + let mut fields = source_schema.fields().to_vec(); + fields.push(Arc::new(ArrowField::new(ROW_ID, DataType::UInt64, false))); + provider_as_source(Arc::new(EmptyTable::new(Arc::new(ArrowSchema::new( + fields, + ))))) } /// The query DataFusion runs: the view's projections plus the group's @@ -180,12 +446,12 @@ pub(super) fn plan( ..definition.clone() }; - let mut fields = source_schema.fields().to_vec(); - fields.push(Arc::new(ArrowField::new(ROW_ID, DataType::UInt64, false))); - let empty = provider_as_source(Arc::new(EmptyTable::new(Arc::new(ArrowSchema::new( - fields, - ))))); - let planned = logical_plan(&session(), empty, &definition)?; + let planned = logical_plan( + &session(HashMap::new()), + empty_source(&source_schema), + &definition, + )?; + ivf_columns(&planned)?; let mut exprs = Vec::new(); planned @@ -241,7 +507,7 @@ pub(super) async fn stream( scanner.project(inputs)?; let scan: SendableRecordBatchStream = scanner.try_into_stream().await?.into(); - let state = session(); + let state = session(bind(source, definition).await?); let ctx = SessionContext::new_with_state(state.clone()); let source = ctx.read_one_shot(scan)?.into_view(); let planned = logical_plan(&state, provider_as_source(source), definition)?; diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 39e1d8b9c..6f19c3c91 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -4489,4 +4489,302 @@ mod tests { .unwrap(); assert!(grouped.input_column("x").is_err()); } + + /// Twenty nonzero vectors in two clusters far apart, ids 0-9 and 10-19. + /// Nonzero so every vector has a cosine direction. + async fn clustered_source(conn: &Connection) -> Table { + use arrow_array::types::Float32Type; + use arrow_array::{FixedSizeListArray, Int32Array}; + + let vectors = FixedSizeListArray::from_iter_primitive::( + (0..20).map(|i| { + let base = if i < 10 { 0.0 } else { 100.0 }; + Some(vec![Some(base + 1.0 + i as f32 * 0.1), Some(base)]) + }), + 2, + ); + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(0..20)) as arrow_array::ArrayRef, + ), + ("vec", Arc::new(vectors) as arrow_array::ArrayRef), + ]) + .unwrap(); + conn.create_table("images", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap() + } + + #[tokio::test] + async fn ivf_partition_groups_by_the_index_on_the_column() { + use crate::index::vector::IvfFlatIndexBuilder; + + const BUCKETS: &str = "SELECT ivf_partition(vec) AS bucket, count(*) AS n, \ + min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(vec)"; + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + let unindexed = declare(&source, "buckets", BUCKETS).await.unwrap_err(); + assert!( + unindexed.to_string().contains("IVF vector index"), + "{unindexed}" + ); + + source + .create_index( + &["vec"], + Index::IvfFlat(IvfFlatIndexBuilder::default().num_partitions(2)), + ) + .execute() + .await + .unwrap(); + let view = declare(&source, "buckets", BUCKETS).await.unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!( + rows(view.table(), &["n", "lo", "hi"]).await, + ["10 0 9", "10 10 19"] + ); + let buckets = rows(view.table(), &["bucket"]).await; + assert_eq!(buckets, ["0", "1"]); + } + + #[tokio::test] + async fn ivf_partition_takes_a_column() { + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + for sql in [ + "SELECT ivf_partition(id) AS b, count(*) AS n FROM images GROUP BY ivf_partition(id)", + "SELECT ivf_partition(vec, vec) AS b, count(*) AS n FROM images \ + GROUP BY ivf_partition(vec, vec)", + ] { + assert!(declare(&source, "v", sql).await.is_err(), "{sql}"); + } + } + + /// A cosine index assigns by L2 over unit vectors, as lance's own index + /// path does; forwarding cosine to the assigner panics. + #[tokio::test] + async fn ivf_partition_supports_cosine_indices() { + use crate::index::vector::IvfFlatIndexBuilder; + + const BUCKETS: &str = "SELECT ivf_partition(vec) AS bucket, count(*) AS n, \ + min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(vec)"; + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + source + .create_index( + &["vec"], + Index::IvfFlat( + IvfFlatIndexBuilder::default() + .num_partitions(2) + .distance_type(crate::DistanceType::Cosine), + ), + ) + .execute() + .await + .unwrap(); + let view = declare(&source, "cosine_buckets", BUCKETS).await.unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!( + rows(view.table(), &["n", "lo", "hi"]).await, + ["10 0 9", "10 10 19"] + ); + } + + /// Segments of one index trained separately carry different centroids; + /// grouping by one of them would bucket the other segments' rows wrongly. + #[tokio::test] + async fn ivf_partition_refuses_segments_with_different_models() { + use arrow_array::types::Float32Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array}; + use lance::index::DatasetIndexExt; + use lance::index::vector::VectorIndexParams; + use lance_index::IndexType; + + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + // A second fragment far from the first, so each segment trains its own centroids. + let far = FixedSizeListArray::from_iter_primitive::( + (0..20).map(|i| Some(vec![Some(-500.0 - i as f32), Some(-500.0)])), + 2, + ); + source + .add( + RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(20..40)) as ArrayRef, + ), + ("vec", Arc::new(far) as ArrayRef), + ]) + .unwrap(), + ) + .execute() + .await + .unwrap(); + let dataset = source.as_native().unwrap().dataset.get().await.unwrap(); + let mut dataset = dataset.as_ref().clone(); + let params = VectorIndexParams::ivf_flat(2, lance_linalg::distance::DistanceType::L2); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + let segment = dataset + .create_index_builder(&["vec"], IndexType::Vector, ¶ms) + .name("shared".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + segments.push(segment); + } + dataset + .commit_existing_index_segments("shared", "vec", segments) + .await + .unwrap(); + assert_eq!(dataset.load_indices().await.unwrap().len(), 2); + + let source = conn.open_table("images").execute().await.unwrap(); + let error = declare( + &source, + "buckets", + "SELECT ivf_partition(vec) AS b, count(*) AS n FROM images GROUP BY ivf_partition(vec)", + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("trained separately"), "{error}"); + } + + /// A vector the assigner cannot place has no bucket: it is grouped under + /// NULL, never under partition 0. + #[tokio::test] + async fn ivf_partition_leaves_an_unassignable_vector_unbucketed() { + use crate::index::vector::IvfFlatIndexBuilder; + use arrow_array::types::Float32Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array}; + + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + source + .create_index( + &["vec"], + Index::IvfFlat( + IvfFlatIndexBuilder::default() + .num_partitions(2) + .distance_type(crate::DistanceType::Cosine), + ), + ) + .execute() + .await + .unwrap(); + // The zero vector has no direction, so cosine cannot place it. + let zero = FixedSizeListArray::from_iter_primitive::( + [Some(vec![Some(0.0), Some(0.0)])], + 2, + ); + source + .add( + RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![20])) as ArrayRef), + ("vec", Arc::new(zero) as ArrayRef), + ]) + .unwrap(), + ) + .execute() + .await + .unwrap(); + let view = declare( + &source, + "buckets", + "SELECT ivf_partition(vec) AS bucket, count(*) AS n, min(id) AS lo, max(id) AS hi \ + FROM images GROUP BY ivf_partition(vec)", + ) + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + // ArrayFormatter renders NULL as the empty string; partition numbers + // are the index's own and not asserted. + let groups = rows(view.table(), &["bucket", "n", "lo", "hi"]).await; + let (unbucketed, bucketed): (Vec<_>, Vec<_>) = + groups.iter().partition(|row| row.starts_with(' ')); + assert_eq!(unbucketed, [" 1 20 20"], "{groups:?}"); + let mut clusters: Vec<&str> = bucketed + .iter() + .map(|row| row.split_once(' ').unwrap().1) + .collect(); + clusters.sort(); + assert_eq!(clusters, ["10 0 9", "10 10 19"], "{groups:?}"); + } + + /// Byte vectors group by a Hamming index, the phash case. The centroids + /// are given, as lance's own tests do: two-means over a handful of + /// hashes can converge with one partition empty. + #[tokio::test] + async fn ivf_partition_groups_byte_vectors_by_hamming() { + use arrow_array::types::UInt8Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array}; + use lance::index::DatasetIndexExt; + use lance::index::vector::VectorIndexParams; + use lance_index::IndexType; + use lance_index::vector::ivf::IvfBuildParams; + + let hashes = FixedSizeListArray::from_iter_primitive::( + (0..20u8).map(|i| { + let base = if i < 10 { 0x00 } else { 0xff }; + Some(vec![ + Some(base ^ (i % 4)), + Some(base), + Some(base), + Some(base), + ]) + }), + 4, + ); + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(0..20)) as ArrayRef, + ), + ("phash", Arc::new(hashes) as ArrayRef), + ]) + .unwrap(); + let conn = connect("memory://").execute().await.unwrap(); + let source = conn + .create_table("images", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let centroids = FixedSizeListArray::from_iter_primitive::( + [Some(vec![Some(0); 4]), Some(vec![Some(0xff); 4])], + 4, + ); + let params = VectorIndexParams::with_ivf_flat_params( + lance_linalg::distance::DistanceType::Hamming, + IvfBuildParams::try_with_centroids(2, Arc::new(centroids)).unwrap(), + ); + let mut dataset = source + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .as_ref() + .clone(); + dataset + .create_index(&["phash"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + let source = conn.open_table("images").execute().await.unwrap(); + + const BUCKETS: &str = "SELECT ivf_partition(phash) AS bucket, count(*) AS n, \ + min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(phash)"; + let view = declare(&source, "buckets", BUCKETS).await.unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!( + rows(view.table(), &["bucket", "n", "lo", "hi"]).await, + ["0 10 0 9", "1 10 10 19"] + ); + } } From a6a6617ee2de210a8c663c113e6510bea7a11bcd Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 22 Sep 2026 17:59:54 +0000 Subject: [PATCH 13/15] =?UTF-8?q?Bump=20version:=200.40.0-beta.5=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 2dfbe973d..8387fff7a 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.5" +current_version = "0.40.0-beta.6" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 9905e8b86..19aae6f16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5756,7 +5756,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.5" +version = "0.40.0-beta.6" dependencies = [ "ahash", "anyhow", @@ -5853,7 +5853,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.5" +version = "0.40.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5878,7 +5878,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.5" +version = "0.40.0-beta.6" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index f6e21c69e..391338d1c 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.5 + 0.40.0-beta.6 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index e38de7a79..4076ad3bb 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.5 + 0.40.0-beta.6 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 845d1ea07..ed9cc9e6d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.5 + 0.40.0-beta.6 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index e2357dfc3..63ed96137 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.5" +version = "0.40.0-beta.6" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index b517a7b8a..16a57a94f 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 25c6698c3..b6801e45f 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index c46f40f96..68b72b6b7 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index fc030ef53..f1898b817 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 4f23a3261..fc7eb31b6 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 0594f5610..145a776f2 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 2c819c574..8c09fc38a 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index b35b5eb28..b2c51694f 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.5", + "version": "0.40.0-beta.6", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 8fcdf8914..40ce3dd84 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.5" +version = "0.40.0-beta.6" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 94857a124..64c675989 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.5" +version = "0.40.0-beta.6" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From dd2539c2cae5e57a1fec0c23944a413177738476 Mon Sep 17 00:00:00 2001 From: Rudra Prasad Bhuyan Date: Wed, 23 Sep 2026 02:36:11 +0530 Subject: [PATCH 14/15] fix(python): convert objects for JSON fields (#4211) ## Summary Closes: #4060 - Convert Python dict/list objects to JSON strings during ingestion. - Support JSON fields nested inside structs and lists. - Cover `add` and `merge_insert` JSON ingestion paths. - Add regression coverage for nested struct and list JSON fields. ## Testing - `git diff --check` - Python compilation passed. - Focused pytest was attempted but could not complete because the native extension build stalled during `uv` bootstrap. --- python/python/lancedb/table.py | 110 +++++++++++++- python/python/tests/test_table.py | 232 ++++++++++++++++++++++++++++++ 2 files changed, 340 insertions(+), 2 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index d9c27ba81..297b4237f 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio import inspect +import json import deprecation import warnings from abc import ABC, abstractmethod @@ -266,12 +267,14 @@ def _into_pyarrow_reader( # convert to list of dict if data is a bunch of LanceModels if isinstance(data[0], LanceModel): - schema = data[0].__class__.to_arrow_schema() + model_schema = data[0].__class__.to_arrow_schema() data = [model_to_dict(d) for d in data] - return pa.Table.from_pylist(data, schema=schema).to_reader() + data = _serialize_json_values(data, schema or model_schema) + return pa.Table.from_pylist(data, schema=model_schema).to_reader() elif isinstance(data[0], pa.RecordBatch): return pa.Table.from_batches(data).to_reader() else: + data = _serialize_json_values(data, schema) return pa.Table.from_pylist(data).to_reader() elif _check_for_pandas(data) and isinstance(data, pd.DataFrame): table = pa.Table.from_pandas(data, preserve_index=False) @@ -619,6 +622,108 @@ def _field_extension_name(field: pa.Field) -> Optional[str]: return extension_name +def _is_json_field(field: pa.Field) -> bool: + return _field_extension_name(field) in ("arrow.json", "lance.json") + + +@dataclass(frozen=True) +class _JsonSerializationPlan: + arrow_field: pa.Field + children: Optional[Dict[str, "_JsonSerializationPlan"]] = None + item: Optional["_JsonSerializationPlan"] = None + + +def _json_serialization_plan(field: pa.Field) -> Optional[_JsonSerializationPlan]: + if _is_json_field(field): + return _JsonSerializationPlan(field) + + if pa.types.is_struct(field.type): + children: Dict[str, _JsonSerializationPlan] = {} + for child_field in field.type: + child_plan = _json_serialization_plan(child_field) + if child_plan is not None: + children[child_field.name] = child_plan + if children: + return _JsonSerializationPlan(field, children=children) + + if _is_list_like(field.type): + item_plan = _json_serialization_plan(field.type.value_field) + if item_plan is not None: + return _JsonSerializationPlan(field, item=item_plan) + + return None + + +def _json_serialization_plans( + schema: pa.Schema, +) -> Dict[str, _JsonSerializationPlan]: + plans: Dict[str, _JsonSerializationPlan] = {} + for field in schema: + plan = _json_serialization_plan(field) + if plan is not None: + plans[field.name] = plan + return plans + + +def _serialize_json_value(value: Any, plan: _JsonSerializationPlan) -> Any: + if value is None or isinstance(value, str): + return value + if _is_json_field(plan.arrow_field): + if isinstance(value, (dict, list)): + return json.dumps(value) + return value + + if plan.children is not None and isinstance(value, dict): + serialized = None + for child_name, child_plan in plan.children.items(): + if child_name not in value: + continue + child_value = _serialize_json_value(value[child_name], child_plan) + if child_value is not value[child_name]: + if serialized is None: + serialized = dict(value) + serialized[child_name] = child_value + return serialized if serialized is not None else value + + if plan.item is not None and isinstance(value, list): + serialized = None + for index, item in enumerate(value): + serialized_item = _serialize_json_value(item, plan.item) + if serialized_item is not item: + if serialized is None: + serialized = list(value) + serialized[index] = serialized_item + return serialized if serialized is not None else value + + return value + + +def _serialize_json_values(data: Any, target_schema: Optional[pa.Schema]) -> Any: + if target_schema is None or not isinstance(data, list): + return data + + plans = _json_serialization_plans(target_schema) + if not plans: + return data + + serialized_rows = [] + for row in data: + if not isinstance(row, dict): + serialized_rows.append(row) + continue + serialized_row = None + for field_name, plan in plans.items(): + if field_name not in row: + continue + value = _serialize_json_value(row[field_name], plan) + if value is not row[field_name]: + if serialized_row is None: + serialized_row = dict(row) + serialized_row[field_name] = value + serialized_rows.append(serialized_row if serialized_row is not None else row) + return serialized_rows + + def _align_field_types( fields: List[pa.Field], target_fields: List[pa.Field], @@ -5720,6 +5825,7 @@ class AsyncTable: """ schema = await self.schema() + data = _serialize_json_values(data, schema) if on_bad_vectors is None: on_bad_vectors = "error" if fill_value is None: diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 0b3dc01c3..5c56fea38 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4,6 +4,7 @@ import ctypes import gc +import json import os import sys import threading @@ -17,6 +18,7 @@ from typing import List from unittest.mock import patch import lancedb +from lancedb import table as table_module from lancedb.dependencies import _PANDAS_AVAILABLE from lancedb.index import BTree, FTS, HnswFlat, HnswPq, HnswSq, IvfPq import numpy as np @@ -3027,6 +3029,236 @@ async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection): assert rows == [{"id": "c", "j": '{"k":3}'}] +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_add_python_objects_to_json_column(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("payload", pa.json_()), + pa.field("metadata", pa.struct([("label", pa.string())])), + pa.field("tags", pa.list_(pa.string())), + ] + ) + table = mem_db.create_table("json_python_objects_add", schema=schema) + table.add( + [ + { + "id": 1, + "payload": {"foo": "bar", "count": 2}, + "metadata": {"label": "dict"}, + "tags": ["a", "b"], + }, + { + "id": 2, + "payload": { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + }, + "metadata": {"label": "nested"}, + "tags": ["c"], + }, + { + "id": 3, + "payload": ["x", "y", "z"], + "metadata": {"label": "list"}, + "tags": ["d", "e"], + }, + { + "id": 4, + "payload": '{"foo": "bar"}', + "metadata": {"label": "string"}, + "tags": ["f"], + }, + { + "id": 5, + "payload": None, + "metadata": {"label": "null"}, + "tags": ["g"], + }, + ] + ) + + rows = {row["id"]: row for row in table.to_arrow().to_pylist()} + assert json.loads(rows[1]["payload"]) == {"foo": "bar", "count": 2} + assert json.loads(rows[2]["payload"]) == { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + } + assert json.loads(rows[3]["payload"]) == ["x", "y", "z"] + assert json.loads(rows[4]["payload"]) == {"foo": "bar"} + assert rows[5]["payload"] is None + assert rows[1]["metadata"] == {"label": "dict"} + assert rows[1]["tags"] == ["a", "b"] + + matched = table.search().where("json_extract(payload, '$.count') = '2'").to_list() + assert [row["id"] for row in matched] == [1] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_merge_insert_python_objects_to_json_column(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("payload", pa.json_()), + pa.field("metadata", pa.struct([("label", pa.string())])), + pa.field("tags", pa.list_(pa.string())), + ] + ) + table = mem_db.create_table("json_python_objects_merge", schema=schema) + table.add( + [ + { + "id": 1, + "payload": {"old": True}, + "metadata": {"label": "old"}, + "tags": ["old"], + } + ] + ) + + table.merge_insert( + "id" + ).when_matched_update_all().when_not_matched_insert_all().execute( + [ + { + "id": 1, + "payload": {"foo": "bar", "count": 2}, + "metadata": {"label": "dict"}, + "tags": ["a", "b"], + }, + { + "id": 2, + "payload": { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + }, + "metadata": {"label": "nested"}, + "tags": ["c"], + }, + { + "id": 3, + "payload": ["x", "y", "z"], + "metadata": {"label": "list"}, + "tags": ["d", "e"], + }, + { + "id": 4, + "payload": '{"foo": "bar"}', + "metadata": {"label": "string"}, + "tags": ["f"], + }, + { + "id": 5, + "payload": None, + "metadata": {"label": "null"}, + "tags": ["g"], + }, + ] + ) + + rows = {row["id"]: row for row in table.to_arrow().to_pylist()} + assert json.loads(rows[1]["payload"]) == {"foo": "bar", "count": 2} + assert json.loads(rows[2]["payload"]) == { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + } + assert json.loads(rows[3]["payload"]) == ["x", "y", "z"] + assert json.loads(rows[4]["payload"]) == {"foo": "bar"} + assert rows[5]["payload"] is None + assert rows[1]["metadata"] == {"label": "dict"} + assert rows[1]["tags"] == ["a", "b"] + + matched = table.search().where("json_extract(payload, '$.count') = '2'").to_list() + assert [row["id"] for row in matched] == [1] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_add_python_objects_to_nested_json_fields(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", + pa.struct([pa.field("payload", pa.json_())]), + ), + pa.field("documents", pa.list_(pa.field("item", pa.json_()))), + ] + ) + table = mem_db.create_table("nested_json_python_objects_add", schema=schema) + + table.add( + [ + { + "id": 1, + "info": {"payload": {"kind": "struct", "value": 1}}, + "documents": [{"kind": "list", "value": 2}], + } + ] + ) + + row = table.to_arrow().to_pylist()[0] + assert json.loads(row["info"]["payload"]) == {"kind": "struct", "value": 1} + assert json.loads(row["documents"][0]) == {"kind": "list", "value": 2} + assert [ + row["id"] + for row in table.search() + .where("json_extract(info.payload, '$.value') = '1'") + .to_list() + ] == [1] + assert [ + row["id"] + for row in table.search() + .where("json_extract(documents[1], '$.value') = '2'") + .to_list() + ] == [1] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_json_serialization_plan_skips_non_json_branches(): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), 3)), + pa.field("payload", pa.json_()), + pa.field( + "metadata", + pa.struct( + [ + pa.field("label", pa.string()), + pa.field("payload", pa.json_()), + ] + ), + ), + pa.field( + "documents", + pa.list_( + pa.struct( + [ + pa.field("title", pa.string()), + pa.field("payload", pa.json_()), + ] + ) + ), + ), + ] + ) + + plans = table_module._json_serialization_plans(schema) + + assert set(plans) == {"payload", "metadata", "documents"} + metadata_children = plans["metadata"].children + assert metadata_children is not None + assert set(metadata_children) == {"payload"} + documents_item = plans["documents"].item + assert documents_item is not None + assert documents_item.children is not None + assert set(documents_item.children) == {"payload"} + + @pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") @pytest.mark.asyncio async def test_add_all_null_json_batch(mem_db_async: AsyncConnection): From 8541d6d9df784b38e391560e7ed1f6d2fc0fed89 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 23 Sep 2026 00:09:40 -0700 Subject: [PATCH 15/15] feat: add view CRUD APIs (#4236) A view is a named query a database stores and plans on every read. It holds no rows, which is the whole difference from a materialized view. ## API | Verb | Route | | --- | --- | | `create_view(name, query, namespace_path)` | `POST /v1/view/{id}/create` | | `describe_view(name, namespace_path)` | `POST /v1/view/{id}/describe` | | `drop_view(name, namespace_path)` | `POST /v1/view/{id}/drop` | | `list_views(namespace_path)` | `GET /v1/namespace/{id}/view/list` | On `Connection` and the `Database` trait, with the remote client, Python (sync and async) and Node bindings. Local databases return `NotSupported`: the server side is Sophon's, where a view is an object of the database manifest. `ViewDescription` carries the defining query, the database *and namespace path* unqualified names in it resolve against, and the schema the query resolved to. `create_view` returns one, so a caller has the schema without a second call. Both defaults travel with the view because it outlives the session that declared it: the server re-plans the stored query on every read, so a reader resolving an unqualified name against its own defaults would read a different table. `default_namespace_path` crosses the wire as `default_namespace`, a path like `namespace`, absent for the root. There is no replace: a name already taken is an error, and changing a view is a drop followed by a create, each authorized against what it actually touches. Querying a view stays SQL's job. There are no rows behind a view, so there is no `open_view` returning a `Table`. --- docs/src/js/classes/Connection.md | 95 +++++++ docs/src/js/globals.md | 1 + docs/src/js/interfaces/ViewDescription.md | 78 ++++++ docs/src/python/python.md | 4 + nodejs/__test__/remote.test.ts | 61 +++++ nodejs/lancedb/connection.ts | 67 +++++ nodejs/lancedb/index.ts | 1 + nodejs/lancedb/view.ts | 48 ++++ nodejs/src/connection.rs | 83 ++++++ python/python/lancedb/__init__.py | 2 + python/python/lancedb/_lancedb.pyi | 12 + python/python/lancedb/db.py | 141 +++++++++++ python/python/lancedb/remote/db.py | 25 ++ python/python/lancedb/view.py | 40 +++ python/python/tests/test_remote_db.py | 54 ++++ python/src/connection.rs | 88 ++++++- rust/lancedb/src/connection.rs | 94 ++++++- rust/lancedb/src/database.rs | 36 +++ rust/lancedb/src/lib.rs | 1 + rust/lancedb/src/remote/db.rs | 291 +++++++++++++++++++++- rust/lancedb/src/utils/mod.rs | 21 ++ rust/lancedb/src/view.rs | 44 ++++ 22 files changed, 1283 insertions(+), 4 deletions(-) create mode 100644 docs/src/js/interfaces/ViewDescription.md create mode 100644 nodejs/lancedb/view.ts create mode 100644 python/python/lancedb/view.py create mode 100644 rust/lancedb/src/view.rs diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index d46d414d3..75af5102e 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -319,6 +319,38 @@ Creates a new Table and initialize it with new data. *** +### createView() + +```ts +abstract createView( + name, + query, + namespacePath?): Promise +``` + +Create a view: a named query the database plans on every read. + +The query is planned once, at creation, so one that cannot be planned is +rejected now rather than at the first read. A view holds no rows, and its +readers see its sources as they are at read time. + +There is no replace: a name already taken is an error, and changing a +view is a drop followed by a create. + +#### Parameters + +* **name**: `string` + +* **query**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<[`ViewDescription`](../interfaces/ViewDescription.md)> + +*** + ### describeNamespace() ```ts @@ -342,6 +374,27 @@ The namespace's properties *** +### describeView() + +```ts +abstract describeView(name, namespacePath?): Promise +``` + +What this database records about the view named `name`: its defining +query and the schema that query resolved to. + +#### Parameters + +* **name**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<[`ViewDescription`](../interfaces/ViewDescription.md)> + +*** + ### display() ```ts @@ -498,6 +551,28 @@ on the returned job to know when cleanup has finished. *** +### dropView() + +```ts +abstract dropView(name, namespacePath?): Promise +``` + +Drop the view named `name`. + +The tables it reads are untouched: a view holds no rows of its own. + +#### Parameters + +* **name**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<`void`> + +*** + ### isOpen() ```ts @@ -636,6 +711,26 @@ A page of table names and an *** +### listViews() + +```ts +abstract listViews(namespacePath?): Promise +``` + +The names of the views in one namespace. + +Names only; a definition comes from [describeView](Connection.md#describeview). + +#### Parameters + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<`string`[]> + +*** + ### openJob() ```ts diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 6d242a15b..2f49578b1 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -149,6 +149,7 @@ - [UpdateOptions](interfaces/UpdateOptions.md) - [UpdateResult](interfaces/UpdateResult.md) - [Version](interfaces/Version.md) +- [ViewDescription](interfaces/ViewDescription.md) - [WriteExecutionOptions](interfaces/WriteExecutionOptions.md) - [WriteProgress](interfaces/WriteProgress.md) diff --git a/docs/src/js/interfaces/ViewDescription.md b/docs/src/js/interfaces/ViewDescription.md new file mode 100644 index 000000000..d53c884b5 --- /dev/null +++ b/docs/src/js/interfaces/ViewDescription.md @@ -0,0 +1,78 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ViewDescription + +# Interface: ViewDescription + +What a database records about one view. + +A view holds no rows: it stores the statement that defines it and the +schema that statement resolved to, so a reader sees its sources as they +are at read time. + +## Properties + +### defaultDatabase + +```ts +defaultDatabase: string; +``` + +The database that unqualified table names in [query](ViewDescription.md#query) resolve against. + +*** + +### defaultNamespacePath + +```ts +defaultNamespacePath: string[]; +``` + +The namespace path those unqualified names resolve against; empty is the +root namespace. + +Recorded with the view because it outlives the session that declared it: +a reader resolving the query against its own default namespace could read +a different table than the view was defined over. + +*** + +### name + +```ts +name: string; +``` + +The view's name within its namespace. + +*** + +### namespacePath + +```ts +namespacePath: string[]; +``` + +The namespace holding the view; empty is the root namespace. + +*** + +### query + +```ts +query: string; +``` + +The defining query, as the database stores it. + +*** + +### schema + +```ts +schema: Schema; +``` + +The schema the defining query resolved to when the view was created. diff --git a/docs/src/python/python.md b/docs/src/python/python.md index d1ba66280..68cd306c4 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -198,6 +198,10 @@ listing a storage directory. ::: lancedb.materialized_view.MaterializedViewDefinition +## Views + +::: lancedb.view.ViewDescription + ## Expressions Type-safe expression builder for filters and projections. Use these instead diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 40f4c67db..4c1d21604 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -100,6 +100,67 @@ describe("remote connection", () => { ); }); + it("creates a view and decodes the schema it resolved to", async () => { + await withMockDatabase( + (req, res) => { + expect(req.method).toBe("POST"); + expect(req.url).toBe("/v1/view/analytics$adults/create"); + res.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + name: "adults", + namespace: ["analytics"], + query: "SELECT name FROM people", + // biome-ignore lint/style/useNamingConvention: the wire field is snake_case + default_database: "db", + // biome-ignore lint/style/useNamingConvention: the wire field is snake_case + default_namespace: ["analytics"], + schema: { + fields: [ + { name: "name", nullable: true, type: { type: "utf8" } }, + ], + }, + }), + ); + }, + async (db) => { + const view = await db.createView("adults", "SELECT name FROM people", [ + "analytics", + ]); + expect(view.name).toBe("adults"); + expect(view.namespacePath).toEqual(["analytics"]); + expect(view.query).toBe("SELECT name FROM people"); + expect(view.defaultDatabase).toBe("db"); + expect(view.defaultNamespacePath).toEqual(["analytics"]); + expect(view.schema.fields.map((f) => f.name)).toEqual(["name"]); + }, + ); + }); + + it("lists and drops views through their own routes", async () => { + await withMockDatabase( + (req, res) => { + expect(req.url).toBe("/v1/namespace/$/view/list"); + res + .writeHead(200, { "content-type": "application/json" }) + .end(JSON.stringify({ views: ["adults"] })); + }, + async (db) => { + expect(await db.listViews()).toEqual(["adults"]); + }, + ); + + await withMockDatabase( + (req, res) => { + expect(req.method).toBe("POST"); + expect(req.url).toBe("/v1/view/adults/drop"); + res.writeHead(200, { "content-type": "application/json" }).end("{}"); + }, + async (db) => { + await db.dropView("adults"); + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index f5a6679cf..9637d60c2 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -31,6 +31,7 @@ import type { ListNamespacesResponse, ListTablesResponse, } from "./native"; +import { ViewDescription, viewDescriptionFromNative } from "./view"; export type { CreateNamespaceResponse, DescribeNamespaceResponse, @@ -377,6 +378,45 @@ export abstract class Connection { namespacePath?: string[], ): Promise; + /** + * Create a view: a named query the database plans on every read. + * + * The query is planned once, at creation, so one that cannot be planned is + * rejected now rather than at the first read. A view holds no rows, and its + * readers see its sources as they are at read time. + * + * There is no replace: a name already taken is an error, and changing a + * view is a drop followed by a create. + */ + abstract createView( + name: string, + query: string, + namespacePath?: string[], + ): Promise; + + /** + * What this database records about the view named `name`: its defining + * query and the schema that query resolved to. + */ + abstract describeView( + name: string, + namespacePath?: string[], + ): Promise; + + /** + * Drop the view named `name`. + * + * The tables it reads are untouched: a view holds no rows of its own. + */ + abstract dropView(name: string, namespacePath?: string[]): Promise; + + /** + * The names of the views in one namespace. + * + * Names only; a definition comes from {@link describeView}. + */ + abstract listViews(namespacePath?: string[]): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -696,6 +736,33 @@ export class LocalConnection extends Connection { ); } + async createView( + name: string, + query: string, + namespacePath?: string[], + ): Promise { + return viewDescriptionFromNative( + await this.inner.createView(name, query, namespacePath ?? []), + ); + } + + async describeView( + name: string, + namespacePath?: string[], + ): Promise { + return viewDescriptionFromNative( + await this.inner.describeView(name, namespacePath ?? []), + ); + } + + async dropView(name: string, namespacePath?: string[]): Promise { + return this.inner.dropView(name, namespacePath ?? []); + } + + async listViews(namespacePath?: string[]): Promise { + return this.inner.listViews(namespacePath ?? []); + } + async listTables( namespacePathOrOptions?: string[] | Partial, options?: Partial, diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index c0a4dfcbd..ed6889232 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -26,6 +26,7 @@ export { MaterializedViewDefinition, MaterializedViewSelect, } from "./materialized_view"; +export { ViewDescription } from "./view"; export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js"; // OpenTelemetry metrics bridge. Only the high-level entry point is public; the diff --git a/nodejs/lancedb/view.ts b/nodejs/lancedb/view.ts new file mode 100644 index 000000000..5ed848648 --- /dev/null +++ b/nodejs/lancedb/view.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { Schema, tableFromIPC } from "apache-arrow"; +import { ViewDescription as NativeViewDescription } from "./native"; + +/** + * What a database records about one view. + * + * A view holds no rows: it stores the statement that defines it and the + * schema that statement resolved to, so a reader sees its sources as they + * are at read time. + */ +export interface ViewDescription { + /** The view's name within its namespace. */ + name: string; + /** The namespace holding the view; empty is the root namespace. */ + namespacePath: string[]; + /** The defining query, as the database stores it. */ + query: string; + /** The database that unqualified table names in {@link query} resolve against. */ + defaultDatabase: string; + /** + * The namespace path those unqualified names resolve against; empty is the + * root namespace. + * + * Recorded with the view because it outlives the session that declared it: + * a reader resolving the query against its own default namespace could read + * a different table than the view was defined over. + */ + defaultNamespacePath: string[]; + /** The schema the defining query resolved to when the view was created. */ + schema: Schema; +} + +/** Decode the schema the binding hands over as an Arrow IPC file. */ +export function viewDescriptionFromNative( + view: NativeViewDescription, +): ViewDescription { + return { + name: view.name, + namespacePath: view.namespacePath, + query: view.query, + defaultDatabase: view.defaultDatabase, + defaultNamespacePath: view.defaultNamespacePath, + schema: tableFromIPC(view.schema).schema, + }; +} diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index 0c07ffd50..ee2a9470b 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -55,6 +55,32 @@ pub struct DropNamespaceResponse { pub transaction_id: Option>, } +/// What a database records about one view. +#[napi(object)] +pub struct ViewDescription { + pub name: String, + pub namespace_path: Vec, + pub query: String, + pub default_database: String, + pub default_namespace_path: Vec, + /// The view's schema as an empty Arrow IPC file, the way a table reports + /// its own. + pub schema: Buffer, +} + +impl ViewDescription { + fn from_inner(view: lancedb::view::ViewDescription) -> napi::Result { + Ok(Self { + name: view.name, + namespace_path: view.namespace_path, + query: view.query, + default_database: view.default_database, + default_namespace_path: view.default_namespace_path, + schema: crate::util::schema_to_buffer(&view.schema)?, + }) + } +} + impl Connection { pub(crate) fn inner_new(inner: LanceDBConnection) -> Self { Self { inner: Some(inner) } @@ -368,6 +394,63 @@ impl Connection { .default_error() } + /// Create a view: a named query planned on every read. + #[napi(catch_unwind)] + pub async fn create_view( + &self, + name: String, + query: String, + namespace_path: Option>, + ) -> napi::Result { + let ns = namespace_path.unwrap_or_default(); + let view = self + .get_inner()? + .create_view(&name, &query, &ns) + .await + .default_error()?; + ViewDescription::from_inner(view) + } + + /// What the database records about one view. + #[napi(catch_unwind)] + pub async fn describe_view( + &self, + name: String, + namespace_path: Option>, + ) -> napi::Result { + let ns = namespace_path.unwrap_or_default(); + let view = self + .get_inner()? + .describe_view(&name, &ns) + .await + .default_error()?; + ViewDescription::from_inner(view) + } + + /// Drop a view. The tables it reads are untouched. + #[napi(catch_unwind)] + pub async fn drop_view( + &self, + name: String, + namespace_path: Option>, + ) -> napi::Result<()> { + let ns = namespace_path.unwrap_or_default(); + self.get_inner()? + .drop_view(&name, &ns) + .await + .default_error() + } + + /// The names of the views in one namespace. + #[napi(catch_unwind)] + pub async fn list_views( + &self, + namespace_path: Option>, + ) -> napi::Result> { + let ns = namespace_path.unwrap_or_default(); + self.get_inner()?.list_views(&ns).await.default_error() + } + /// Start dropping a materialized view and return its cleanup job. #[napi(catch_unwind)] pub async fn drop_materialized_view_async( diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 362e0cdbf..d34b541a8 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -47,6 +47,7 @@ from .materialized_view import ( MaterializedView, MaterializedViewDefinition, ) +from .view import ViewDescription as ViewDescription from .table import AsyncTable, Table from .types import BaseTokenizerType from ._lancedb import Session @@ -582,6 +583,7 @@ __all__ = [ "AsyncMaterializedView", "MaterializedView", "MaterializedViewDefinition", + "ViewDescription", "connect", "connect_async", "tokenize", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 6d873850d..ba4e3a0df 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -171,6 +171,18 @@ class Connection(object): async def describe_secret( self, name: str, namespace_path: Optional[List[str]] = None ) -> Tuple[str, int, int]: ... + async def create_view( + self, name: str, query: str, namespace_path: Optional[List[str]] = None + ) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ... + async def describe_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ... + async def drop_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def list_views( + self, namespace_path: Optional[List[str]] = None + ) -> List[str]: ... async def list_jobs(self) -> List[JobInfo]: ... async def cancel_job(self, job_id: str) -> bool: ... async def execute_query_async( diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index b0db22532..10d52c63e 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -75,6 +75,7 @@ from .util import ( get_uri_scheme, validate_table_name, ) +from .view import ViewDescription import deprecation @@ -96,6 +97,28 @@ from .namespace_utils import ( ) +def _view_description( + described: Tuple[str, List[str], str, str, List[str], "pa.Schema"], +) -> ViewDescription: + """Name the fields the binding returns positionally.""" + ( + name, + namespace_path, + query, + default_database, + default_namespace_path, + schema, + ) = described + return ViewDescription( + name=name, + query=query, + default_database=default_database, + schema=schema, + namespace_path=namespace_path, + default_namespace_path=default_namespace_path, + ) + + class DBConnection(EnforceOverrides): """An active LanceDB connection interface.""" @@ -918,6 +941,68 @@ class DBConnection(EnforceOverrides): "Secret operations are not supported for this connection type" ) + def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """Create a view: a named query the database plans on every read. + + The query is planned once, at creation, so one that cannot be planned + is refused now rather than at the first read. A view holds no rows, and + its readers see its sources as they are at read time. + + There is no replace: a name already taken is an error, and changing a + view is a drop followed by a create. Local connections raise + ``NotImplementedError``. + + >>> import lancedb + >>> db = lancedb.connect("db://my_database") # doctest: +SKIP + >>> view = db.create_view( + ... "adults", "SELECT name FROM people WHERE age >= 18" + ... ) # doctest: +SKIP + >>> view.schema # doctest: +SKIP + name: string + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + + def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """What this database records about a view: its defining query and the + schema that query resolved to. + + The schema is the one recorded at creation; a source altered since then + shows up when the view is read. Local connections raise + ``NotImplementedError``. + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + + def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a view. + + The tables it reads are untouched: a view holds no rows of its own. + Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + + def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + """The names of the views in one namespace. + + Names only; a definition comes from + [describe_view][lancedb.db.DBConnection.describe_view]. Local + connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + def open_job(self, job_id: str) -> Job: """Open a server-side job by id, returning a handle with its record already populated. @@ -1731,6 +1816,30 @@ class LanceDBConnection(DBConnection): ) -> SecretInfo: return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path)) + @override + def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run( + self._conn.create_view(name, query, namespace_path=namespace_path) + ) + + @override + def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run(self._conn.describe_view(name, namespace_path=namespace_path)) + + @override + def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_view(name, namespace_path=namespace_path)) + + @override + def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_views(namespace_path=namespace_path)) + @override def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" @@ -2689,6 +2798,38 @@ class AsyncConnection(object): updated_at_millis=updated_at_millis, ) + async def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """Create a view: a named query the database plans on every read. + + See + [DBConnection.create_view][lancedb.DBConnection.create_view]. + """ + return _view_description( + await self._inner.create_view(name, query, list(namespace_path or [])) + ) + + async def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """What this database records about a view: query and schema.""" + return _view_description( + await self._inner.describe_view(name, list(namespace_path or [])) + ) + + async def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a view. The tables it reads are untouched.""" + await self._inner.drop_view(name, list(namespace_path or [])) + + async def list_views( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[str]: + """The names of the views in one namespace.""" + return await self._inner.list_views(list(namespace_path or [])) + async def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" return await self._inner.list_jobs() diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index ef40ef007..962ef66e1 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -41,6 +41,7 @@ from ..sql import Query as SqlQuery from ..sql import QueryDescription from ..materialized_view import MaterializedView, SelectArg from ..secrets import EnvVarSecret, SecretInfo +from ..view import ViewDescription if TYPE_CHECKING: from .._lancedb import JobInfo @@ -910,6 +911,30 @@ class RemoteDBConnection(DBConnection): ) -> None: LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path)) + @override + def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run( + self._conn.create_view(name, query, namespace_path=namespace_path) + ) + + @override + def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run(self._conn.describe_view(name, namespace_path=namespace_path)) + + @override + def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_view(name, namespace_path=namespace_path)) + + @override + def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_views(namespace_path=namespace_path)) + @override def list_jobs(self) -> List["JobInfo"]: """List server-side jobs across the database's tables.""" diff --git a/python/python/lancedb/view.py b/python/python/lancedb/view.py new file mode 100644 index 000000000..715252fc6 --- /dev/null +++ b/python/python/lancedb/view.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Views: a named query a database stores and plans on every read. + +A view holds no rows. What it stores is the statement that defines it and the +schema that statement resolved to, so a reader sees the sources as they are +now. See ``DBConnection.create_view``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List + +if TYPE_CHECKING: + import pyarrow as pa + + +@dataclass(frozen=True) +class ViewDescription: + """What a database records about one view.""" + + name: str + """The view's name within its namespace.""" + query: str + """The defining query, as the database stores it.""" + default_database: str + """The database that unqualified table names in ``query`` resolve against.""" + schema: "pa.Schema" + """The schema the defining query resolved to when the view was created.""" + namespace_path: List[str] = field(default_factory=list) + """The namespace holding the view; empty is the root namespace.""" + default_namespace_path: List[str] = field(default_factory=list) + """The namespace path those unqualified names resolve against. + + Recorded with the view because it outlives the session that declared it: a + reader resolving the query against its own default namespace could read a + different table than the view was defined over. + """ diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 1c887b218..c03862a49 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2747,3 +2747,57 @@ def test_remote_job_handle_reports_its_own_detail(): "limit": 500, "filter": "state = 'claim_complete'", } + + +def test_view_crud_addresses_its_own_routes(): + # The view verbs are their own routes, and the schema comes back in the + # namespace spec's JSON encoding, decoded into a pyarrow schema. + paths = [] + + def handler(request): + paths.append((request.command, request.path)) + if request.path.endswith("/view/list"): + body = {"views": ["adults"]} + elif request.path.endswith("/drop"): + body = {} + else: + body = { + "name": "adults", + "namespace": ["analytics"], + "query": "SELECT name FROM people", + "default_database": "dev", + "default_namespace": ["analytics"], + "schema": { + "fields": [ + {"name": "name", "nullable": True, "type": {"type": "utf8"}} + ] + }, + } + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(body).encode()) + + with mock_lancedb_connection(handler) as db: + view = db.create_view( + "adults", "SELECT name FROM people", namespace_path=["analytics"] + ) + assert view.name == "adults" + assert view.namespace_path == ["analytics"] + assert view.query == "SELECT name FROM people" + assert view.default_database == "dev" + assert view.default_namespace_path == ["analytics"] + assert view.schema == pa.schema([pa.field("name", pa.utf8(), nullable=True)]) + + described = db.describe_view("adults", namespace_path=["analytics"]) + assert described.schema == view.schema + + assert db.list_views(namespace_path=["analytics"]) == ["adults"] + db.drop_view("adults", namespace_path=["analytics"]) + + assert paths == [ + ("POST", "/v1/view/analytics$adults/create"), + ("POST", "/v1/view/analytics$adults/describe"), + ("GET", "/v1/namespace/analytics/view/list"), + ("POST", "/v1/view/analytics$adults/drop"), + ] diff --git a/python/src/connection.rs b/python/src/connection.rs index 3385e6b2d..69f54c683 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -13,7 +13,11 @@ use crate::{ runtime::future_into_py, table::Table, }; -use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow}; +use arrow::{ + datatypes::Schema, + ffi_stream::ArrowArrayStreamReader, + pyarrow::{FromPyArrow, ToPyArrow}, +}; use lancedb::{ connection::Connection as LanceConnection, connection::NamespaceClientPushdownOperation, @@ -100,6 +104,28 @@ fn parse_default_namespace_path(path: Option>) -> PyResult, String, String, Vec, Py); + +/// A view description as a plain tuple, with the schema converted to the +/// pyarrow schema the caller would get from any other lancedb API. The Python +/// layer names the fields; this keeps the binding free of a class that would +/// have to be kept in step with the Rust struct. +fn view_description_to_py(view: lancedb::view::ViewDescription) -> PyResult { + Python::attach(|py| { + let schema = view.schema.to_pyarrow(py)?.unbind(); + Ok(( + view.name, + view.namespace_path, + view.query, + view.default_database, + view.default_namespace_path, + schema, + )) + }) +} + #[pymethods] impl Connection { fn __repr__(&self) -> String { @@ -862,6 +888,66 @@ impl Connection { }) } + #[pyo3(signature = (name, query, namespace_path=None))] + pub fn create_view( + self_: PyRef<'_, Self>, + name: String, + query: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + let view = inner + .create_view(name, query, &namespace_path) + .await + .infer_error()?; + view_description_to_py(view) + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn describe_view( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + let view = inner + .describe_view(name, &namespace_path) + .await + .infer_error()?; + view_description_to_py(view) + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn drop_view( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.drop_view(name, &namespace_path).await.infer_error() + }) + } + + #[pyo3(signature = (namespace_path=None))] + pub fn list_views( + self_: PyRef<'_, Self>, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.list_views(&namespace_path).await.infer_error() + }) + } + pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index d43f8f744..962e9e586 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -37,7 +37,11 @@ use crate::remote::{ }, }; use crate::secrets::SecretInfo; -use crate::utils::{validate_secret_component, validate_secret_reference}; +use crate::utils::{ + validate_namespace, validate_secret_component, validate_secret_reference, + validate_view_reference, +}; +use crate::view::ViewDescription; use lance::io::ObjectStoreParams; pub use lance_file::version::LanceFileVersion; #[cfg(feature = "remote")] @@ -747,6 +751,94 @@ impl Connection { .await } + /// Create a view: a named query the database plans on every read. + /// + /// The query is planned once, here, so one that cannot be planned is + /// refused now rather than at the first read. A view holds no rows, and + /// its readers see its sources as they are at read time. + /// + /// There is no replace: a name already taken is an error, and changing a + /// view is a drop followed by a create, each authorized against what it + /// actually touches. Local databases return [`Error::NotSupported`]. + /// + /// # Example + /// + /// ```no_run + /// # async fn view_lifecycle( + /// # connection: &lancedb::Connection, + /// # ) -> Result<(), Box> { + /// let namespace = vec!["analytics".to_string()]; + /// + /// let view = connection + /// .create_view( + /// "recent_orders", + /// "SELECT id, total FROM orders WHERE total > 100", + /// &namespace, + /// ) + /// .await?; + /// println!("{} has {} columns", view.name, view.schema.fields().len()); + /// + /// // The query comes back as it was recorded, with the defaults its + /// // unqualified names resolve against. + /// let described = connection.describe_view("recent_orders", &namespace).await?; + /// println!("{} in {:?}", described.query, described.default_namespace_path); + /// + /// let names = connection.list_views(&namespace).await?; + /// assert!(names.iter().any(|name| name == "recent_orders")); + /// + /// // Dropping the view leaves `orders` untouched. + /// connection.drop_view("recent_orders", &namespace).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn create_view( + &self, + name: impl AsRef, + query: impl AsRef, + namespace_path: &[String], + ) -> Result { + validate_view_reference(name.as_ref(), namespace_path)?; + self.internal + .create_view(name.as_ref(), query.as_ref(), namespace_path) + .await + } + + /// What this database records about one view: its defining query and the + /// schema that query resolved to. + /// + /// The schema is the one recorded at creation. A source altered since then + /// shows up when the view is read, not here. Local databases return + /// [`Error::NotSupported`]. + pub async fn describe_view( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result { + validate_view_reference(name.as_ref(), namespace_path)?; + self.internal + .describe_view(name.as_ref(), namespace_path) + .await + } + + /// Drop a view. + /// + /// The tables it reads are untouched: a view holds no rows of its own. + /// Local databases return [`Error::NotSupported`]. + pub async fn drop_view(&self, name: impl AsRef, namespace_path: &[String]) -> Result<()> { + validate_view_reference(name.as_ref(), namespace_path)?; + self.internal.drop_view(name.as_ref(), namespace_path).await + } + + /// The names of the views in one namespace. + /// + /// Names only; a definition is query metadata and comes from + /// [`Self::describe_view`]. The client walks all server pages before + /// returning. Local databases return [`Error::NotSupported`]. + pub async fn list_views(&self, namespace_path: &[String]) -> Result> { + validate_namespace(namespace_path)?; + self.internal.list_views(namespace_path).await + } + /// Rename a table in the database. /// /// This is only supported in LanceDB Cloud. diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 91b200043..086d1df35 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -32,6 +32,7 @@ use crate::job::Job; use crate::materialized_view::CreateMaterializedViewRequest; use crate::secrets::SecretInfo; use crate::table::{BaseTable, WriteOptions}; +use crate::view::ViewDescription; pub mod listing; pub mod namespace; @@ -258,6 +259,12 @@ fn secret_catalog_not_supported() -> Result { }) } +fn view_ops_not_supported() -> Result { + Err(crate::error::Error::NotSupported { + message: "View operations are not supported by this database".to_string(), + }) +} + /// The `Database` trait defines the interface for database implementations. /// /// A database is responsible for managing tables and their metadata. @@ -443,6 +450,35 @@ pub trait Database: async fn describe_secret(&self, _name: &str, _namespace_path: &[String]) -> Result { secret_catalog_not_supported() } + /// Create a view from a defining query. The query is planned once, by the + /// database, so one that cannot be planned is refused now rather than at + /// the first read. + async fn create_view( + &self, + _name: &str, + _query: &str, + _namespace_path: &[String], + ) -> Result { + view_ops_not_supported() + } + /// What the database records about one view: its defining query and the + /// schema that query resolved to. + async fn describe_view( + &self, + _name: &str, + _namespace_path: &[String], + ) -> Result { + view_ops_not_supported() + } + /// Drop a view. Its sources are untouched -- a view holds no rows of its + /// own. + async fn drop_view(&self, _name: &str, _namespace_path: &[String]) -> Result<()> { + view_ops_not_supported() + } + /// The names of the views in one namespace. + async fn list_views(&self, _namespace_path: &[String]) -> Result> { + view_ops_not_supported() + } /// Open a job by id, returning a handle with its record already /// populated. Fails with [`crate::Error::JobNotFound`] when the server has /// no such job. diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 4d9afc7c7..530328cbb 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -202,6 +202,7 @@ pub mod table; #[cfg(test)] pub mod test_utils; pub mod utils; +pub mod view; use std::{fmt::Display, str::FromStr}; diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 7a59db665..893c584e1 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -14,8 +14,8 @@ use reqwest::header::CONTENT_TYPE; use lance_namespace::models::{ CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest, - DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest, - ListNamespacesResponse, ListTablesRequest, ListTablesResponse, + DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, JsonArrowSchema, + ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest, ListTablesResponse, }; use crate::Error; @@ -36,6 +36,7 @@ use crate::secrets::SecretBinding; use crate::secrets::SecretInfo; use crate::table::BaseTable; use crate::utils::{reject_relative_segment, validate_table_name}; +use crate::view::ViewDescription; use super::client::{ ClientConfig, HeaderProvider, HttpSend, ID_DELIMITER, RequestResultExt, RestfulLanceDbClient, @@ -784,6 +785,59 @@ struct RemoteListedSecret { name: String, } +/// Define a view from a query. The name and its namespace are the path +/// identifier, so neither appears here. +#[derive(serde::Serialize)] +struct RemoteCreateViewRequest<'a> { + query: &'a str, +} + +/// What the service reports about one view. The schema arrives as the +/// namespace spec's JSON encoding, which is what `describe_table` uses too. +#[derive(serde::Deserialize)] +struct RemoteViewDescription { + name: String, + #[serde(default)] + namespace: Vec, + query: String, + default_database: String, + /// A path, like `namespace`: the root is the absent field rather than a + /// spelling of its own. + #[serde(default)] + default_namespace: Vec, + schema: JsonArrowSchema, +} + +impl RemoteViewDescription { + fn into_description(self, request_id: String) -> Result { + let schema = + lance_namespace::schema::convert_json_arrow_schema(&self.schema).map_err(|source| { + Error::Http { + source: format!("View '{}' has an undecodable schema: {source}", self.name) + .into(), + request_id, + status_code: None, + } + })?; + Ok(ViewDescription { + name: self.name, + namespace_path: self.namespace, + query: self.query, + default_database: self.default_database, + default_namespace_path: self.default_namespace, + schema: Arc::new(schema), + }) + } +} + +#[derive(serde::Deserialize)] +struct RemoteListViewsResponse { + #[serde(default)] + views: Vec, + #[serde(default)] + page_token: Option, +} + /// Bound on `list_jobs` page walking; a warning is logged when the listing /// is truncated at this many pages. const MAX_LIST_JOBS_PAGES: usize = 100; @@ -1122,6 +1176,79 @@ impl Database for RemoteDatabase { response.json().await.err_to_http(request_id) } + async fn create_view( + &self, + name: &str, + query: &str, + namespace_path: &[String], + ) -> Result { + let view_id = build_object_identifier("View name", name, namespace_path)?; + let req = self + .client + .post(&format!("/v1/view/{view_id}/create")) + .json(&RemoteCreateViewRequest { query }); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let description: RemoteViewDescription = + response.json().await.err_to_http(request_id.clone())?; + description.into_description(request_id) + } + + async fn describe_view( + &self, + name: &str, + namespace_path: &[String], + ) -> Result { + let view_id = build_object_identifier("View name", name, namespace_path)?; + let req = self.client.post(&format!("/v1/view/{view_id}/describe")); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let description: RemoteViewDescription = + response.json().await.err_to_http(request_id.clone())?; + description.into_description(request_id) + } + + async fn drop_view(&self, name: &str, namespace_path: &[String]) -> Result<()> { + let view_id = build_object_identifier("View name", name, namespace_path)?; + let req = self.client.post(&format!("/v1/view/{view_id}/drop")); + let (request_id, response) = self.client.send(req).await?; + self.client.check_response(&request_id, response).await?; + Ok(()) + } + + async fn list_views(&self, namespace_path: &[String]) -> Result> { + let namespace_id = build_namespace_identifier(namespace_path)?; + let path = format!("/v1/namespace/{namespace_id}/view/list"); + let mut views = Vec::new(); + let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); + loop { + let mut req = self.client.get(&path); + if let Some(token) = &page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let response: RemoteListViewsResponse = + response.json().await.err_to_http(request_id.clone())?; + views.extend(response.views); + let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "View listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); + } + page_token = Some(next_page_token); + } + Ok(views) + } + async fn open_job(&self, job_id: &str) -> Result { let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string()); match crate::job::JobHandle::describe(&handle).await { @@ -3798,6 +3925,166 @@ mod tests { .unwrap(); } + /// A view description carries the schema in the namespace spec's JSON + /// encoding, so the body a test asserts against is the one that encoding + /// produces rather than a hand-written guess at it. + fn view_description_body(name: &str, namespace: &[&str], query: &str) -> String { + let schema = Schema::new(vec![Field::new("id", DataType::Int32, true)]); + serde_json::json!({ + "name": name, + "namespace": namespace, + "query": query, + "default_database": "db", + "default_namespace": namespace, + "schema": lance_namespace::schema::arrow_schema_to_json(&schema).unwrap(), + }) + .to_string() + } + + #[tokio::test] + async fn test_create_view_posts_the_query_and_returns_the_resolved_schema() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::POST); + assert_eq!(request.url().path(), "/v1/view/analytics$adults/create"); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + // The name and its namespace address the view in the path, so the + // body says only what they cannot. + assert_eq!(body, serde_json::json!({"query": "SELECT id FROM people"})); + http::Response::builder() + .status(200) + .body(view_description_body( + "adults", + &["analytics"], + "SELECT id FROM people", + )) + .unwrap() + }); + let view = conn + .create_view("adults", "SELECT id FROM people", &["analytics".into()]) + .await + .unwrap(); + assert_eq!(view.name, "adults"); + assert_eq!(view.namespace_path, vec!["analytics".to_string()]); + assert_eq!(view.query, "SELECT id FROM people"); + assert_eq!(view.default_database, "db"); + assert_eq!(view.default_namespace_path, vec!["analytics".to_string()]); + assert_eq!(view.schema.field(0).name(), "id"); + } + + #[tokio::test] + async fn test_describe_and_drop_address_the_view_in_the_path() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/view/adults/describe"); + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder() + .status(200) + .body(view_description_body( + "adults", + &[], + "SELECT id FROM people", + )) + .unwrap() + }); + let view = conn.describe_view("adults", &[]).await.unwrap(); + assert!(view.namespace_path.is_empty()); + assert_eq!(view.schema.fields().len(), 1); + + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::POST); + assert_eq!(request.url().path(), "/v1/view/analytics$adults/drop"); + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_view("adults", &["analytics".into()]) + .await + .unwrap(); + } + + /// A schema the client cannot decode is a broken response, not a view + /// with no columns: reporting it as an error keeps a caller from reading + /// an empty schema as the truth about the view. + #[tokio::test] + async fn test_describe_view_rejects_an_undecodable_schema() { + let conn = Connection::new_with_handler(|_| { + http::Response::builder() + .status(200) + .body( + r#"{"name":"adults","query":"SELECT 1","default_database":"db", + "schema":{"fields":[{"name":"id","type":{"type":"nonesuch"}, + "nullable":true}]}}"#, + ) + .unwrap() + }); + let error = conn.describe_view("adults", &[]).await.unwrap_err(); + assert!(error.to_string().contains("undecodable schema"), "{error}"); + } + + #[tokio::test] + async fn test_list_views_walks_pages() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/analytics/view/list"); + let page = request + .url() + .query_pairs() + .find(|(key, _)| key == "page_token") + .map(|(_, value)| value.into_owned()); + let body = match page.as_deref() { + None => r#"{"views":[],"page_token":"p2"}"#, + Some("p2") => r#"{"views":["adults"]}"#, + Some(other) => panic!("unexpected page token: {other}"), + }; + http::Response::builder().status(200).body(body).unwrap() + }); + assert_eq!( + conn.list_views(&["analytics".into()]).await.unwrap(), + vec!["adults".to_string()] + ); + } + + /// A server that keeps handing back the same token would otherwise spin + /// forever. + #[tokio::test] + async fn test_list_views_rejects_a_repeated_page_token() { + let conn = Connection::new_with_handler(|_| { + http::Response::builder() + .status(200) + .body(r#"{"views":["adults"],"page_token":"same"}"#) + .unwrap() + }); + let error = conn.list_views(&[]).await.unwrap_err(); + assert!( + error.to_string().contains("repeated a page_token"), + "{error}" + ); + } + + /// A name carrying the delimiter would split back apart as a different + /// view, so it is refused before it reaches a route. + #[tokio::test] + async fn test_view_names_that_would_resplit_are_refused() { + let conn = Connection::new_with_handler(|_| -> http::Response { + panic!("an invalid identifier must not reach the service") + }); + for (name, namespace) in [ + ("analytics$adults", vec![]), + ("adults", vec!["ana$lytics".to_string()]), + ("", vec![]), + ("..", vec![]), + ] { + let error = match conn.describe_view(name, &namespace).await { + Ok(_) => panic!("accepted {name:?} in {namespace:?}"), + Err(error) => error.to_string(), + }; + // A view is not a table, so the refusal says so. + assert!( + error.contains("view name") || error.contains("view namespace path segment"), + "{error}" + ); + } + } + #[tokio::test] async fn test_create_function_async_sends_canonical_request_and_decodes_typed_job() { const REQUEST: &str = include_str!( diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index f80cc876a..52b7bd5e8 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -174,6 +174,27 @@ pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Resul validate_secret_component("Secret name", name) } +/// Validate a view name and every segment of the namespace path holding it. +/// +/// Worded for a view rather than deferring to [`validate_table_name`]: a view +/// is not a table, and a caller who mistypes one should not be told their +/// table name is invalid. The rule is the same one every object name follows, +/// which is what lets the `$`-joined identifier split back apart. +pub fn validate_view_reference(name: &str, namespace_path: &[String]) -> Result<()> { + for segment in namespace_path { + validate_view_component("view namespace path segment", segment)?; + } + validate_view_component("view name", name) +} + +/// Validate one component of a view identifier: a view name, or one segment of +/// the namespace path holding it. +pub fn validate_view_component(what: &str, value: &str) -> Result<()> { + check_object_name(value).map_err(|reason| Error::InvalidInput { + message: format!("invalid {what} '{value}': {reason}"), + }) +} + /// Validate all components of a namespace /// /// Iterates through all namespace components and validates each one. diff --git a/rust/lancedb/src/view.rs b/rust/lancedb/src/view.rs new file mode 100644 index 000000000..ca9703209 --- /dev/null +++ b/rust/lancedb/src/view.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Views: a named query a database stores and plans on every read. +//! +//! A view holds no rows. What it stores is the statement that defines it and +//! the schema that statement resolved to, so a reader sees the sources as they +//! are now rather than as they were when the view was created. That is the +//! whole difference from [`crate::materialized_view`], which holds rows and +//! moves them forward by refresh. +//! +//! The verbs live on [`crate::connection::Connection`]. A view is read through +//! SQL, by name, so there is no `open_view` returning a +//! [`crate::table::Table`]: there would be no rows behind it. + +use arrow_schema::SchemaRef; + +/// What a database records about one view. +/// +/// Returned by [`crate::connection::Connection::describe_view`], and by +/// `create_view` so a caller has the resolved schema without a second call. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ViewDescription { + /// The view's name within its namespace. + pub name: String, + /// The namespace holding the view; empty is the root namespace. + pub namespace_path: Vec, + /// The defining query, as the database stores it. + pub query: String, + /// The database that unqualified table names in `query` resolve against. + /// A view created from a session connected elsewhere can name a database + /// other than its own, so this is part of what the query means rather + /// than a restatement of where the view lives. + pub default_database: String, + /// The namespace path those unqualified names resolve against; empty is + /// the root namespace. + /// + /// Recorded with the view because it outlives the session that declared + /// it: a reader that resolved the query against its own default namespace + /// could read a different table than the view was defined over. + pub default_namespace_path: Vec, + /// The schema the defining query resolved to when the view was created. + pub schema: SchemaRef, +}