rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 14:52:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/origin/main' into gatekeeper/fix-2325-1

Browse Source 
# Conflicts:
#	python/python/lancedb/__init__.py
This commit is contained in:
Gatefixer
2026-09-23 07:30:26 +00:00
parent 0576203078 8541d6d9df
commit 8d8ba5d157
58 changed files with 3947 additions and 273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.3"
current_version = "0.40.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
.github/workflows/pypi-publish.yml
+2 -6
View File
@@ -129,12 +129,8 @@ jobs:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
# Fat LTO of the cdylib is single-threaded and the peak-memory step of the
# build. ThinLTO parallelizes it across the runner's cores, at some cost
# to runtime performance on our least performance-sensitive platform.
# Matches what the nodejs Windows builds already do in npm-publish.yml.
CARGO_PROFILE_RELEASE_LTO: thin
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16
# Keep the repository's fat LTO / 1 codegen unit release profile to leave
# headroom below PyPI's file-size limit for Windows wheels.
steps:
- uses: actions/checkout@v6
with:
CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
Cargo.lock
Generated
+162 -71
View File
@@ -141,7 +141,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -1819,7 +1819,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -2937,6 +2937,12 @@ dependencies = [
"sqlparser 0.62.0",
]
[[package]]
name = "datasketches"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d"
[[package]]
name = "debugid"
version = "0.8.0"
@@ -3081,7 +3087,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3309,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3499,6 +3505,65 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "foyer-common"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30"
dependencies = [
"anyhow",
"bytes",
"cfg-if 1.0.4",
"foyer-tokio",
"mixtrics",
"parking_lot",
"pin-project",
"twox-hash",
]
[[package]]
name = "foyer-intrusive-collections"
version = "0.10.0-dev"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b"
dependencies = [
"memoffset",
]
[[package]]
name = "foyer-memory"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82"
dependencies = [
"anyhow",
"asyncband",
"bitflags 2.11.1",
"datasketches",
"equivalent",
"foyer-common",
"foyer-intrusive-collections",
"foyer-tokio",
"futures-util",
"hashbrown 0.17.1",
"itertools 0.15.0",
"mixtrics",
"parking_lot",
"paste",
"pin-project",
"serde",
"tracing",
]
[[package]]
name = "foyer-tokio"
version = "0.22.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918"
dependencies = [
"tokio",
]
[[package]]
name = "frostem"
version = "1.20260804.1"
@@ -3523,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4004,18 +4069,23 @@ dependencies = [
[[package]]
name = "goosefs-sdk"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f521e3beacc809ced36a8aff4a468b1bf61302c8f4dca41c3f21ac0557c22ed"
checksum = "1e471f7bce40936cf2023855c7350e31e11b3f703eb7cb2ce73c9ffc451f87d2"
dependencies = [
"arc-swap",
"async-trait",
"bytes",
"dashmap",
"fastrand",
"foyer-common",
"foyer-memory",
"futures",
"hostname",
"io-uring",
"itoa",
"libc",
"lru",
"prost",
"rand 0.9.5",
"serde",
@@ -4468,7 +4538,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.3",
"socket2 0.5.10",
"system-configuration",
"tokio",
"tower-service",
@@ -4810,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -5073,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arc-swap",
"arrow",
@@ -5146,8 +5216,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5169,7 +5239,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5183,7 +5253,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5192,8 +5262,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrayref",
"crunchy",
@@ -5203,8 +5273,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5241,8 +5311,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5273,8 +5343,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5291,8 +5361,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"proc-macro2",
"quote",
@@ -5301,8 +5371,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5335,8 +5405,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5368,8 +5438,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"datafusion",
"geo-traits",
@@ -5383,8 +5453,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arc-swap",
"arrow",
@@ -5452,8 +5522,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5475,8 +5545,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5488,6 +5558,7 @@ dependencies = [
"bytes",
"chrono",
"futures",
"goosefs-sdk",
"http 1.5.0",
"io-uring",
"lance-core",
@@ -5498,6 +5569,7 @@ dependencies = [
"object_store 0.14.1",
"object_store_opendal",
"opendal",
"opendal-service-goosefs",
"path_abs",
"pin-project",
"prost",
@@ -5516,8 +5588,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5531,8 +5603,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow",
"async-trait",
@@ -5546,8 +5618,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5600,8 +5672,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5615,8 +5687,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow",
"arrow-array",
@@ -5656,8 +5728,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5670,8 +5742,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "13.0.0-beta.6"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5684,7 +5756,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.40.0-beta.3"
version = "0.40.0-beta.6"
dependencies = [
"ahash",
"anyhow",
@@ -5781,7 +5853,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.40.0-beta.3"
version = "0.40.0-beta.6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5806,7 +5878,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.40.0-beta.3"
version = "0.40.0-beta.6"
dependencies = [
"arc-swap",
"arrow",
@@ -6207,6 +6279,15 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "memoffset"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
dependencies = [
"autocfg",
]
[[package]]
name = "metrics"
version = "0.24.6"
@@ -6280,6 +6361,16 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "mixtrics"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9"
dependencies = [
"itertools 0.15.0",
"parking_lot",
]
[[package]]
name = "moka"
version = "0.12.16"
@@ -6540,7 +6631,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -6669,7 +6760,7 @@ version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d"
dependencies = [
"base64 0.22.1",
"base64 0.21.7",
"chrono",
"getrandom 0.2.17",
"http 1.5.0",
@@ -8043,8 +8134,8 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
"heck 0.4.1",
"itertools 0.11.0",
"log",
"multimap",
"petgraph",
@@ -8063,7 +8154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.11.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -8285,7 +8376,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls 0.23.45",
"socket2 0.6.3",
"socket2 0.5.10",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -8323,7 +8414,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.3",
"socket2 0.5.10",
"tracing",
"windows-sys 0.60.2",
]
@@ -9123,7 +9214,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9194,7 +9285,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9741,7 +9832,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9753,7 +9844,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9776,7 +9867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -9895,7 +9986,7 @@ dependencies = [
"cfg-if 1.0.4",
"libc",
"psm",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -10187,7 +10278,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -11208,7 +11299,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.3</version>
<version>0.40.0-beta.6</version>
</dependency>
```
docs/src/js/classes/Connection.md
+95
View File
@@ -319,6 +319,38 @@ Creates a new Table and initialize it with new data.
***
### createView()
```ts
abstract createView(
name,
query,
namespacePath?): Promise<ViewDescription>
```
Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned is
rejected now rather than at the first read. A view holds no rows, and its
readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create.
#### Parameters
* **name**: `string`
* **query**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### describeNamespace()
```ts
@@ -342,6 +374,27 @@ The namespace's properties
***
### describeView()
```ts
abstract describeView(name, namespacePath?): Promise<ViewDescription>
```
What this database records about the view named `name`: its defining
query and the schema that query resolved to.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;[`ViewDescription`](../interfaces/ViewDescription.md)&gt;
***
### display()
```ts
@@ -498,6 +551,28 @@ on the returned job to know when cleanup has finished.
***
### dropView()
```ts
abstract dropView(name, namespacePath?): Promise<void>
```
Drop the view named `name`.
The tables it reads are untouched: a view holds no rows of its own.
#### Parameters
* **name**: `string`
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`void`&gt;
***
### isOpen()
```ts
@@ -636,6 +711,26 @@ A page of table names and an
***
### listViews()
```ts
abstract listViews(namespacePath?): Promise<string[]>
```
The names of the views in one namespace.
Names only; a definition comes from [describeView](Connection.md#describeview).
#### Parameters
* **namespacePath?**: `string`[]
#### Returns
`Promise`&lt;`string`[]&gt;
***
### openJob()
```ts
docs/src/js/globals.md
+1
View File
@@ -149,6 +149,7 @@
- [UpdateOptions](interfaces/UpdateOptions.md)
- [UpdateResult](interfaces/UpdateResult.md)
- [Version](interfaces/Version.md)
- [ViewDescription](interfaces/ViewDescription.md)
- [WriteExecutionOptions](interfaces/WriteExecutionOptions.md)
- [WriteProgress](interfaces/WriteProgress.md)
docs/src/js/interfaces/MaterializedViewDefinition.md
+2 -2
View File
@@ -8,8 +8,8 @@
The query that defines a materialized view, as stored:
`SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row
per element it returns.
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
position yields one row per element it returns.
## Properties
docs/src/js/interfaces/ViewDescription.md
+78
View File
@@ -0,0 +1,78 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / ViewDescription
# Interface: ViewDescription
What a database records about one view.
A view holds no rows: it stores the statement that defines it and the
schema that statement resolved to, so a reader sees its sources as they
are at read time.
## Properties
### defaultDatabase
```ts
defaultDatabase: string;
```
The database that unqualified table names in [query](ViewDescription.md#query) resolve against.
***
### defaultNamespacePath
```ts
defaultNamespacePath: string[];
```
The namespace path those unqualified names resolve against; empty is the
root namespace.
Recorded with the view because it outlives the session that declared it:
a reader resolving the query against its own default namespace could read
a different table than the view was defined over.
***
### name
```ts
name: string;
```
The view's name within its namespace.
***
### namespacePath
```ts
namespacePath: string[];
```
The namespace holding the view; empty is the root namespace.
***
### query
```ts
query: string;
```
The defining query, as the database stores it.
***
### schema
```ts
schema: Schema<any>;
```
The schema the defining query resolved to when the view was created.
docs/src/python/python.md
+4
View File
@@ -200,6 +200,10 @@ listing a storage directory.
::: lancedb.materialized_view.MaterializedViewDefinition
## Views
::: lancedb.view.ViewDescription
## Expressions
Type-safe expression builder for filters and projections. Use these instead
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.3</version>
<version>0.40.0-beta.6</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.3</version>
<version>0.40.0-beta.6</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>13.0.0-beta.6</lance-core.version>
<lance-core.version>13.0.0-beta.8</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.40.0-beta.3"
version = "0.40.0-beta.6"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/materialized_view.test.ts
+1 -1
View File
@@ -54,7 +54,7 @@ describe("materialized views", () => {
// A newer writer's layout is reported, never guessed at.
for (const newer of [
`{"format":2,"query":${JSON.stringify(query)}}`,
`{"format":3,"query":${JSON.stringify(query)}}`,
'{"kind":"select_v3","source_table":"people"}',
]) {
expect(() => read(newer)).toThrow(/cannot refresh/);
nodejs/__test__/remote.test.ts
+61
View File
@@ -100,6 +100,67 @@ describe("remote connection", () => {
);
});
it("creates a view and decodes the schema it resolved to", async () => {
await withMockDatabase(
(req, res) => {
expect(req.method).toBe("POST");
expect(req.url).toBe("/v1/view/analytics$adults/create");
res.writeHead(200, { "content-type": "application/json" }).end(
JSON.stringify({
name: "adults",
namespace: ["analytics"],
query: "SELECT name FROM people",
// biome-ignore lint/style/useNamingConvention: the wire field is snake_case
default_database: "db",
// biome-ignore lint/style/useNamingConvention: the wire field is snake_case
default_namespace: ["analytics"],
schema: {
fields: [
{ name: "name", nullable: true, type: { type: "utf8" } },
],
},
}),
);
},
async (db) => {
const view = await db.createView("adults", "SELECT name FROM people", [
"analytics",
]);
expect(view.name).toBe("adults");
expect(view.namespacePath).toEqual(["analytics"]);
expect(view.query).toBe("SELECT name FROM people");
expect(view.defaultDatabase).toBe("db");
expect(view.defaultNamespacePath).toEqual(["analytics"]);
expect(view.schema.fields.map((f) => f.name)).toEqual(["name"]);
},
);
});
it("lists and drops views through their own routes", async () => {
await withMockDatabase(
(req, res) => {
expect(req.url).toBe("/v1/namespace/$/view/list");
res
.writeHead(200, { "content-type": "application/json" })
.end(JSON.stringify({ views: ["adults"] }));
},
async (db) => {
expect(await db.listViews()).toEqual(["adults"]);
},
);
await withMockDatabase(
(req, res) => {
expect(req.method).toBe("POST");
expect(req.url).toBe("/v1/view/adults/drop");
res.writeHead(200, { "content-type": "application/json" }).end("{}");
},
async (db) => {
await db.dropView("adults");
},
);
});
it("should accept partial connection options", async () => {
await connect("db://test", {
apiKey: "fake",
nodejs/lancedb/connection.ts
+67
View File
@@ -31,6 +31,7 @@ import type {
ListNamespacesResponse,
ListTablesResponse,
} from "./native";
import { ViewDescription, viewDescriptionFromNative } from "./view";
export type {
CreateNamespaceResponse,
DescribeNamespaceResponse,
@@ -377,6 +378,45 @@ export abstract class Connection {
namespacePath?: string[],
): Promise<Job>;
/**
* Create a view: a named query the database plans on every read.
*
* The query is planned once, at creation, so one that cannot be planned is
* rejected now rather than at the first read. A view holds no rows, and its
* readers see its sources as they are at read time.
*
* There is no replace: a name already taken is an error, and changing a
* view is a drop followed by a create.
*/
abstract createView(
name: string,
query: string,
namespacePath?: string[],
): Promise<ViewDescription>;
/**
* What this database records about the view named `name`: its defining
* query and the schema that query resolved to.
*/
abstract describeView(
name: string,
namespacePath?: string[],
): Promise<ViewDescription>;
/**
* Drop the view named `name`.
*
* The tables it reads are untouched: a view holds no rows of its own.
*/
abstract dropView(name: string, namespacePath?: string[]): Promise<void>;
/**
* The names of the views in one namespace.
*
* Names only; a definition comes from {@link describeView}.
*/
abstract listViews(namespacePath?: string[]): Promise<string[]>;
abstract openTable(
name: string,
namespacePath?: string[],
@@ -696,6 +736,33 @@ export class LocalConnection extends Connection {
);
}
async createView(
name: string,
query: string,
namespacePath?: string[],
): Promise<ViewDescription> {
return viewDescriptionFromNative(
await this.inner.createView(name, query, namespacePath ?? []),
);
}
async describeView(
name: string,
namespacePath?: string[],
): Promise<ViewDescription> {
return viewDescriptionFromNative(
await this.inner.describeView(name, namespacePath ?? []),
);
}
async dropView(name: string, namespacePath?: string[]): Promise<void> {
return this.inner.dropView(name, namespacePath ?? []);
}
async listViews(namespacePath?: string[]): Promise<string[]> {
return this.inner.listViews(namespacePath ?? []);
}
async listTables(
namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
options?: Partial<ListTablesOptions>,
nodejs/lancedb/index.ts
+1
View File
@@ -26,6 +26,7 @@ export {
MaterializedViewDefinition,
MaterializedViewSelect,
} from "./materialized_view";
export { ViewDescription } from "./view";
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
nodejs/lancedb/materialized_view.ts
+7 -4
View File
@@ -7,14 +7,17 @@ import { Table } from "./table";
/** Schema metadata key holding a materialized view's definition. */
export const DEFINITION_META_KEY = "mv.definition";
/** The stored layout this version reads: `{"format": 1, "query": "<SQL>"}`. */
export const DEFINITION_FORMAT = 1;
/**
* The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
* format 2 being a query with `GROUP BY`.
*/
export const DEFINITION_FORMAT = 2;
/**
* The query that defines a materialized view, as stored:
* `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
* [WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row
* per element it returns.
* [WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM`
* position yields one row per element it returns.
*/
export interface MaterializedViewDefinition {
/** The defining query, in the canonical spelling the server stores. */
nodejs/lancedb/view.ts
+48
View File
@@ -0,0 +1,48 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Schema, tableFromIPC } from "apache-arrow";
import { ViewDescription as NativeViewDescription } from "./native";
/**
* What a database records about one view.
*
* A view holds no rows: it stores the statement that defines it and the
* schema that statement resolved to, so a reader sees its sources as they
* are at read time.
*/
export interface ViewDescription {
/** The view's name within its namespace. */
name: string;
/** The namespace holding the view; empty is the root namespace. */
namespacePath: string[];
/** The defining query, as the database stores it. */
query: string;
/** The database that unqualified table names in {@link query} resolve against. */
defaultDatabase: string;
/**
* The namespace path those unqualified names resolve against; empty is the
* root namespace.
*
* Recorded with the view because it outlives the session that declared it:
* a reader resolving the query against its own default namespace could read
* a different table than the view was defined over.
*/
defaultNamespacePath: string[];
/** The schema the defining query resolved to when the view was created. */
schema: Schema;
}
/** Decode the schema the binding hands over as an Arrow IPC file. */
export function viewDescriptionFromNative(
view: NativeViewDescription,
): ViewDescription {
return {
name: view.name,
namespacePath: view.namespacePath,
query: view.query,
defaultDatabase: view.defaultDatabase,
defaultNamespacePath: view.defaultNamespacePath,
schema: tableFromIPC(view.schema).schema,
};
}
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.40.0-beta.3",
"version": "0.40.0-beta.6",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/connection.rs
+83
View File
@@ -55,6 +55,32 @@ pub struct DropNamespaceResponse {
pub transaction_id: Option<Vec<String>>,
}
/// What a database records about one view.
#[napi(object)]
pub struct ViewDescription {
pub name: String,
pub namespace_path: Vec<String>,
pub query: String,
pub default_database: String,
pub default_namespace_path: Vec<String>,
/// The view's schema as an empty Arrow IPC file, the way a table reports
/// its own.
pub schema: Buffer,
}
impl ViewDescription {
fn from_inner(view: lancedb::view::ViewDescription) -> napi::Result<Self> {
Ok(Self {
name: view.name,
namespace_path: view.namespace_path,
query: view.query,
default_database: view.default_database,
default_namespace_path: view.default_namespace_path,
schema: crate::util::schema_to_buffer(&view.schema)?,
})
}
}
impl Connection {
pub(crate) fn inner_new(inner: LanceDBConnection) -> Self {
Self { inner: Some(inner) }
@@ -368,6 +394,63 @@ impl Connection {
.default_error()
}
/// Create a view: a named query planned on every read.
#[napi(catch_unwind)]
pub async fn create_view(
&self,
name: String,
query: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<ViewDescription> {
let ns = namespace_path.unwrap_or_default();
let view = self
.get_inner()?
.create_view(&name, &query, &ns)
.await
.default_error()?;
ViewDescription::from_inner(view)
}
/// What the database records about one view.
#[napi(catch_unwind)]
pub async fn describe_view(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<ViewDescription> {
let ns = namespace_path.unwrap_or_default();
let view = self
.get_inner()?
.describe_view(&name, &ns)
.await
.default_error()?;
ViewDescription::from_inner(view)
}
/// Drop a view. The tables it reads are untouched.
#[napi(catch_unwind)]
pub async fn drop_view(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?
.drop_view(&name, &ns)
.await
.default_error()
}
/// The names of the views in one namespace.
#[napi(catch_unwind)]
pub async fn list_views(
&self,
namespace_path: Option<Vec<String>>,
) -> napi::Result<Vec<String>> {
let ns = namespace_path.unwrap_or_default();
self.get_inner()?.list_views(&ns).await.default_error()
}
/// Start dropping a materialized view and return its cleanup job.
#[napi(catch_unwind)]
pub async fn drop_materialized_view_async(
python/CLAUDE.md
-1
View File
@@ -1 +0,0 @@
AGENTS.md
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.40.0-beta.3"
version = "0.40.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/__init__.py
+2
View File
@@ -47,6 +47,7 @@ from .materialized_view import (
MaterializedView,
MaterializedViewDefinition,
)
from .view import ViewDescription as ViewDescription
from .table import AsyncTable, CompactionOptions, Table
from .types import BaseTokenizerType
from ._lancedb import Session
@@ -582,6 +583,7 @@ __all__ = [
"AsyncMaterializedView",
"MaterializedView",
"MaterializedViewDefinition",
"ViewDescription",
"connect",
"connect_async",
"tokenize",
python/python/lancedb/_lancedb.pyi
+15
View File
@@ -153,6 +153,9 @@ class Connection(object):
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def drop_function_async(
self, name: str, version: str
) -> Tuple[bool, Job]: ...
async def create_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
) -> None: ...
@@ -168,6 +171,18 @@ class Connection(object):
async def describe_secret(
self, name: str, namespace_path: Optional[List[str]] = None
) -> Tuple[str, int, int]: ...
async def create_view(
self, name: str, query: str, namespace_path: Optional[List[str]] = None
) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ...
async def describe_view(
self, name: str, namespace_path: Optional[List[str]] = None
) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ...
async def drop_view(
self, name: str, namespace_path: Optional[List[str]] = None
) -> None: ...
async def list_views(
self, namespace_path: Optional[List[str]] = None
) -> List[str]: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def execute_query_async(
python/python/lancedb/db.py
+179 -6
View File
@@ -18,6 +18,7 @@ from typing import (
Literal,
Optional,
Sequence,
Tuple,
Union,
)
from uuid import UUID
@@ -74,6 +75,7 @@ from .util import (
get_uri_scheme,
validate_table_name,
)
from .view import ViewDescription
import deprecation
@@ -95,6 +97,28 @@ from .namespace_utils import (
)
def _view_description(
described: Tuple[str, List[str], str, str, List[str], "pa.Schema"],
) -> ViewDescription:
"""Name the fields the binding returns positionally."""
(
name,
namespace_path,
query,
default_database,
default_namespace_path,
schema,
) = described
return ViewDescription(
name=name,
query=query,
default_database=default_database,
schema=schema,
namespace_path=namespace_path,
default_namespace_path=default_namespace_path,
)
class DBConnection(EnforceOverrides):
"""An active LanceDB connection interface."""
@@ -828,12 +852,26 @@ class DBConnection(EnforceOverrides):
)
def drop_function(self, name: str, *, version: str) -> bool:
"""Remove the current Function name binding from the remote catalog.
"""Drop a Function name and the object it was bound to.
The requested version must exist in the currently named object.
Object history and existing computed-column references are retained.
Returns True when the name was removed and False when it was absent.
Local connections raise NotImplementedError.
The requested version must exist in the currently named object. Returns
True when the name was removed and False when it was absent. The
object's content is deleted with it, which may finish after this
returns; use :meth:`drop_function_async` to wait for that. Local
connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
"""Drop a Function name and return its cleanup job.
The name is unbound before this returns; the object's content may still
be being deleted. Call :meth:`Job.wait` to wait for that to finish. When
the server deletes inline, or when nothing was bound, the returned job
is already finished and has no id. Local connections raise
NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
@@ -903,6 +941,68 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
"""Create a view: a named query the database plans on every read.
The query is planned once, at creation, so one that cannot be planned
is refused now rather than at the first read. A view holds no rows, and
its readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create. Local connections raise
``NotImplementedError``.
>>> import lancedb
>>> db = lancedb.connect("db://my_database") # doctest: +SKIP
>>> view = db.create_view(
... "adults", "SELECT name FROM people WHERE age >= 18"
... ) # doctest: +SKIP
>>> view.schema # doctest: +SKIP
name: string
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
)
def describe_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
"""What this database records about a view: its defining query and the
schema that query resolved to.
The schema is the one recorded at creation; a source altered since then
shows up when the view is read. Local connections raise
``NotImplementedError``.
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
)
def drop_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Drop a view.
The tables it reads are untouched: a view holds no rows of its own.
Local connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
)
def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
"""The names of the views in one namespace.
Names only; a definition comes from
[describe_view][lancedb.db.DBConnection.describe_view]. Local
connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"View operations are not supported for this connection type"
)
def open_job(self, job_id: str) -> Job:
"""Open a server-side job by id, returning a handle with its record
already populated.
@@ -1683,6 +1783,11 @@ class LanceDBConnection(DBConnection):
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
return dropped, Job(job)
@override
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
@@ -1711,6 +1816,30 @@ class LanceDBConnection(DBConnection):
) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
@override
def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
return LOOP.run(
self._conn.create_view(name, query, namespace_path=namespace_path)
)
@override
def describe_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
return LOOP.run(self._conn.describe_view(name, namespace_path=namespace_path))
@override
def drop_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.drop_view(name, namespace_path=namespace_path))
@override
def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_views(namespace_path=namespace_path))
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
@@ -2596,9 +2725,21 @@ class AsyncConnection(object):
]
async def drop_function(self, name: str, *, version: str) -> bool:
"""Remove the current name binding, retaining the object and its history."""
"""Drop a Function name and the object it was bound to."""
return await self._inner.drop_function(name, version)
async def drop_function_async(
self, name: str, *, version: str
) -> Tuple[bool, AsyncJob]:
"""Drop a Function name and return its cleanup job.
The name is unbound before this returns; the object's content may still
be being deleted. Await :meth:`AsyncJob.wait` to wait for that to
finish.
"""
dropped, job = await self._inner.drop_function_async(name, version)
return dropped, AsyncJob(job)
async def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
@@ -2657,6 +2798,38 @@ class AsyncConnection(object):
updated_at_millis=updated_at_millis,
)
async def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
"""Create a view: a named query the database plans on every read.
See
[DBConnection.create_view][lancedb.DBConnection.create_view].
"""
return _view_description(
await self._inner.create_view(name, query, list(namespace_path or []))
)
async def describe_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
"""What this database records about a view: query and schema."""
return _view_description(
await self._inner.describe_view(name, list(namespace_path or []))
)
async def drop_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Drop a view. The tables it reads are untouched."""
await self._inner.drop_view(name, list(namespace_path or []))
async def list_views(
self, *, namespace_path: Optional[List[str]] = None
) -> List[str]:
"""The names of the views in one namespace."""
return await self._inner.list_views(list(namespace_path or []))
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
python/python/lancedb/materialized_view.py
+5 -4
View File
@@ -29,9 +29,10 @@ SelectArg = Union[
]
DEFINITION_FORMAT = 1
"""The stored layout this version reads: ``{"format": 1, "query": "<SQL>"}``.
A ``kind`` key beside it is for readers older than the format number."""
DEFINITION_FORMAT = 2
"""The newest stored layout this version reads: ``{"format": N, "query": "<SQL>"}``,
format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for
readers older than the format number."""
@dataclass
@@ -40,7 +41,7 @@ class MaterializedViewDefinition:
SELECT columns
FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
[WHERE predicate] [LIMIT n]
[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]
A Function in ``FROM`` position yields one row per element it returns.
"""
python/python/lancedb/remote/db.py
+31
View File
@@ -16,6 +16,7 @@ from typing import (
List,
Optional,
Sequence,
Tuple,
Union,
)
from urllib.parse import urlparse
@@ -40,6 +41,7 @@ from ..sql import Query as SqlQuery
from ..sql import QueryDescription
from ..materialized_view import MaterializedView, SelectArg
from ..secrets import EnvVarSecret, SecretInfo
from ..view import ViewDescription
if TYPE_CHECKING:
from .._lancedb import JobInfo
@@ -876,6 +878,11 @@ class RemoteDBConnection(DBConnection):
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]:
dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version))
return dropped, Job(job)
@override
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
@@ -904,6 +911,30 @@ class RemoteDBConnection(DBConnection):
) -> None:
LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
@override
def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
return LOOP.run(
self._conn.create_view(name, query, namespace_path=namespace_path)
)
@override
def describe_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> ViewDescription:
return LOOP.run(self._conn.describe_view(name, namespace_path=namespace_path))
@override
def drop_view(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.drop_view(name, namespace_path=namespace_path))
@override
def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_views(namespace_path=namespace_path))
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
python/python/lancedb/remote/table.py
+3
View File
@@ -984,6 +984,9 @@ class RemoteTable(Table):
self,
transforms: Dict[str, str | FunctionApplication]
| FunctionApplication
| pa.Field
| List[pa.Field]
| pa.Schema
| None = None,
*,
computed: Dict[str, str] | None = None,
python/python/lancedb/table.py
+108 -2
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import inspect
import json
import deprecation
import warnings
from abc import ABC, abstractmethod
@@ -352,12 +353,14 @@ def _into_pyarrow_reader(
# convert to list of dict if data is a bunch of LanceModels
if isinstance(data[0], LanceModel):
schema = data[0].__class__.to_arrow_schema()
model_schema = data[0].__class__.to_arrow_schema()
data = [model_to_dict(d) for d in data]
return pa.Table.from_pylist(data, schema=schema).to_reader()
data = _serialize_json_values(data, schema or model_schema)
return pa.Table.from_pylist(data, schema=model_schema).to_reader()
elif isinstance(data[0], pa.RecordBatch):
return pa.Table.from_batches(data).to_reader()
else:
data = _serialize_json_values(data, schema)
return pa.Table.from_pylist(data).to_reader()
elif _check_for_pandas(data) and isinstance(data, pd.DataFrame):
table = pa.Table.from_pandas(data, preserve_index=False)
@@ -705,6 +708,108 @@ def _field_extension_name(field: pa.Field) -> Optional[str]:
return extension_name
def _is_json_field(field: pa.Field) -> bool:
return _field_extension_name(field) in ("arrow.json", "lance.json")
@dataclass(frozen=True)
class _JsonSerializationPlan:
arrow_field: pa.Field
children: Optional[Dict[str, "_JsonSerializationPlan"]] = None
item: Optional["_JsonSerializationPlan"] = None
def _json_serialization_plan(field: pa.Field) -> Optional[_JsonSerializationPlan]:
if _is_json_field(field):
return _JsonSerializationPlan(field)
if pa.types.is_struct(field.type):
children: Dict[str, _JsonSerializationPlan] = {}
for child_field in field.type:
child_plan = _json_serialization_plan(child_field)
if child_plan is not None:
children[child_field.name] = child_plan
if children:
return _JsonSerializationPlan(field, children=children)
if _is_list_like(field.type):
item_plan = _json_serialization_plan(field.type.value_field)
if item_plan is not None:
return _JsonSerializationPlan(field, item=item_plan)
return None
def _json_serialization_plans(
schema: pa.Schema,
) -> Dict[str, _JsonSerializationPlan]:
plans: Dict[str, _JsonSerializationPlan] = {}
for field in schema:
plan = _json_serialization_plan(field)
if plan is not None:
plans[field.name] = plan
return plans
def _serialize_json_value(value: Any, plan: _JsonSerializationPlan) -> Any:
if value is None or isinstance(value, str):
return value
if _is_json_field(plan.arrow_field):
if isinstance(value, (dict, list)):
return json.dumps(value)
return value
if plan.children is not None and isinstance(value, dict):
serialized = None
for child_name, child_plan in plan.children.items():
if child_name not in value:
continue
child_value = _serialize_json_value(value[child_name], child_plan)
if child_value is not value[child_name]:
if serialized is None:
serialized = dict(value)
serialized[child_name] = child_value
return serialized if serialized is not None else value
if plan.item is not None and isinstance(value, list):
serialized = None
for index, item in enumerate(value):
serialized_item = _serialize_json_value(item, plan.item)
if serialized_item is not item:
if serialized is None:
serialized = list(value)
serialized[index] = serialized_item
return serialized if serialized is not None else value
return value
def _serialize_json_values(data: Any, target_schema: Optional[pa.Schema]) -> Any:
if target_schema is None or not isinstance(data, list):
return data
plans = _json_serialization_plans(target_schema)
if not plans:
return data
serialized_rows = []
for row in data:
if not isinstance(row, dict):
serialized_rows.append(row)
continue
serialized_row = None
for field_name, plan in plans.items():
if field_name not in row:
continue
value = _serialize_json_value(row[field_name], plan)
if value is not row[field_name]:
if serialized_row is None:
serialized_row = dict(row)
serialized_row[field_name] = value
serialized_rows.append(serialized_row if serialized_row is not None else row)
return serialized_rows
def _align_field_types(
fields: List[pa.Field],
target_fields: List[pa.Field],
@@ -5819,6 +5924,7 @@ class AsyncTable:
"""
schema = await self.schema()
data = _serialize_json_values(data, schema)
if on_bad_vectors is None:
on_bad_vectors = "error"
if fill_value is None:
python/python/lancedb/view.py
+40
View File
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Views: a named query a database stores and plans on every read.
A view holds no rows. What it stores is the statement that defines it and the
schema that statement resolved to, so a reader sees the sources as they are
now. See ``DBConnection.create_view``.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, List
if TYPE_CHECKING:
import pyarrow as pa
@dataclass(frozen=True)
class ViewDescription:
"""What a database records about one view."""
name: str
"""The view's name within its namespace."""
query: str
"""The defining query, as the database stores it."""
default_database: str
"""The database that unqualified table names in ``query`` resolve against."""
schema: "pa.Schema"
"""The schema the defining query resolved to when the view was created."""
namespace_path: List[str] = field(default_factory=list)
"""The namespace holding the view; empty is the root namespace."""
default_namespace_path: List[str] = field(default_factory=list)
"""The namespace path those unqualified names resolve against.
Recorded with the view because it outlives the session that declared it: a
reader resolving the query against its own default namespace could read a
different table than the view was defined over.
"""
python/python/tests/test_materialized_views.py
+1 -1
View File
@@ -522,7 +522,7 @@ def test_stored_queries_and_legacy_layouts_are_read():
# A newer writer's layout is reported, never guessed at.
for newer in (
{"format": 2, "query": query},
{"format": 3, "query": query},
{"kind": "select_v3", "source_table": "people"},
):
with pytest.raises(NotImplementedError, match="cannot refresh"):
python/python/tests/test_remote_db.py
+54
View File
@@ -2747,3 +2747,57 @@ def test_remote_job_handle_reports_its_own_detail():
"limit": 500,
"filter": "state = 'claim_complete'",
}
def test_view_crud_addresses_its_own_routes():
# The view verbs are their own routes, and the schema comes back in the
# namespace spec's JSON encoding, decoded into a pyarrow schema.
paths = []
def handler(request):
paths.append((request.command, request.path))
if request.path.endswith("/view/list"):
body = {"views": ["adults"]}
elif request.path.endswith("/drop"):
body = {}
else:
body = {
"name": "adults",
"namespace": ["analytics"],
"query": "SELECT name FROM people",
"default_database": "dev",
"default_namespace": ["analytics"],
"schema": {
"fields": [
{"name": "name", "nullable": True, "type": {"type": "utf8"}}
]
},
}
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(body).encode())
with mock_lancedb_connection(handler) as db:
view = db.create_view(
"adults", "SELECT name FROM people", namespace_path=["analytics"]
)
assert view.name == "adults"
assert view.namespace_path == ["analytics"]
assert view.query == "SELECT name FROM people"
assert view.default_database == "dev"
assert view.default_namespace_path == ["analytics"]
assert view.schema == pa.schema([pa.field("name", pa.utf8(), nullable=True)])
described = db.describe_view("adults", namespace_path=["analytics"])
assert described.schema == view.schema
assert db.list_views(namespace_path=["analytics"]) == ["adults"]
db.drop_view("adults", namespace_path=["analytics"])
assert paths == [
("POST", "/v1/view/analytics$adults/create"),
("POST", "/v1/view/analytics$adults/describe"),
("GET", "/v1/namespace/analytics/view/list"),
("POST", "/v1/view/analytics$adults/drop"),
]
python/python/tests/test_table.py
+232
View File
@@ -4,6 +4,7 @@
import ctypes
import gc
import json
import os
import sys
import threading
@@ -17,6 +18,7 @@ from typing import Any, List
from unittest.mock import patch
import lancedb
from lancedb import table as table_module
from lancedb.dependencies import _PANDAS_AVAILABLE
from lancedb.index import BTree, FTS, HnswFlat, HnswPq, HnswSq, IvfPq
import numpy as np
@@ -3027,6 +3029,236 @@ async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection):
assert rows == [{"id": "c", "j": '{"k":3}'}]
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
def test_add_python_objects_to_json_column(mem_db: DBConnection):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("payload", pa.json_()),
pa.field("metadata", pa.struct([("label", pa.string())])),
pa.field("tags", pa.list_(pa.string())),
]
)
table = mem_db.create_table("json_python_objects_add", schema=schema)
table.add(
[
{
"id": 1,
"payload": {"foo": "bar", "count": 2},
"metadata": {"label": "dict"},
"tags": ["a", "b"],
},
{
"id": 2,
"payload": {
"name": "alice",
"tags": ["x", "y"],
"nested": {"enabled": True},
},
"metadata": {"label": "nested"},
"tags": ["c"],
},
{
"id": 3,
"payload": ["x", "y", "z"],
"metadata": {"label": "list"},
"tags": ["d", "e"],
},
{
"id": 4,
"payload": '{"foo": "bar"}',
"metadata": {"label": "string"},
"tags": ["f"],
},
{
"id": 5,
"payload": None,
"metadata": {"label": "null"},
"tags": ["g"],
},
]
)
rows = {row["id"]: row for row in table.to_arrow().to_pylist()}
assert json.loads(rows[1]["payload"]) == {"foo": "bar", "count": 2}
assert json.loads(rows[2]["payload"]) == {
"name": "alice",
"tags": ["x", "y"],
"nested": {"enabled": True},
}
assert json.loads(rows[3]["payload"]) == ["x", "y", "z"]
assert json.loads(rows[4]["payload"]) == {"foo": "bar"}
assert rows[5]["payload"] is None
assert rows[1]["metadata"] == {"label": "dict"}
assert rows[1]["tags"] == ["a", "b"]
matched = table.search().where("json_extract(payload, '$.count') = '2'").to_list()
assert [row["id"] for row in matched] == [1]
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
def test_merge_insert_python_objects_to_json_column(mem_db: DBConnection):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("payload", pa.json_()),
pa.field("metadata", pa.struct([("label", pa.string())])),
pa.field("tags", pa.list_(pa.string())),
]
)
table = mem_db.create_table("json_python_objects_merge", schema=schema)
table.add(
[
{
"id": 1,
"payload": {"old": True},
"metadata": {"label": "old"},
"tags": ["old"],
}
]
)
table.merge_insert(
"id"
).when_matched_update_all().when_not_matched_insert_all().execute(
[
{
"id": 1,
"payload": {"foo": "bar", "count": 2},
"metadata": {"label": "dict"},
"tags": ["a", "b"],
},
{
"id": 2,
"payload": {
"name": "alice",
"tags": ["x", "y"],
"nested": {"enabled": True},
},
"metadata": {"label": "nested"},
"tags": ["c"],
},
{
"id": 3,
"payload": ["x", "y", "z"],
"metadata": {"label": "list"},
"tags": ["d", "e"],
},
{
"id": 4,
"payload": '{"foo": "bar"}',
"metadata": {"label": "string"},
"tags": ["f"],
},
{
"id": 5,
"payload": None,
"metadata": {"label": "null"},
"tags": ["g"],
},
]
)
rows = {row["id"]: row for row in table.to_arrow().to_pylist()}
assert json.loads(rows[1]["payload"]) == {"foo": "bar", "count": 2}
assert json.loads(rows[2]["payload"]) == {
"name": "alice",
"tags": ["x", "y"],
"nested": {"enabled": True},
}
assert json.loads(rows[3]["payload"]) == ["x", "y", "z"]
assert json.loads(rows[4]["payload"]) == {"foo": "bar"}
assert rows[5]["payload"] is None
assert rows[1]["metadata"] == {"label": "dict"}
assert rows[1]["tags"] == ["a", "b"]
matched = table.search().where("json_extract(payload, '$.count') = '2'").to_list()
assert [row["id"] for row in matched] == [1]
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
def test_add_python_objects_to_nested_json_fields(mem_db: DBConnection):
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field(
"info",
pa.struct([pa.field("payload", pa.json_())]),
),
pa.field("documents", pa.list_(pa.field("item", pa.json_()))),
]
)
table = mem_db.create_table("nested_json_python_objects_add", schema=schema)
table.add(
[
{
"id": 1,
"info": {"payload": {"kind": "struct", "value": 1}},
"documents": [{"kind": "list", "value": 2}],
}
]
)
row = table.to_arrow().to_pylist()[0]
assert json.loads(row["info"]["payload"]) == {"kind": "struct", "value": 1}
assert json.loads(row["documents"][0]) == {"kind": "list", "value": 2}
assert [
row["id"]
for row in table.search()
.where("json_extract(info.payload, '$.value') = '1'")
.to_list()
] == [1]
assert [
row["id"]
for row in table.search()
.where("json_extract(documents[1], '$.value') = '2'")
.to_list()
] == [1]
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
def test_json_serialization_plan_skips_non_json_branches():
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 3)),
pa.field("payload", pa.json_()),
pa.field(
"metadata",
pa.struct(
[
pa.field("label", pa.string()),
pa.field("payload", pa.json_()),
]
),
),
pa.field(
"documents",
pa.list_(
pa.struct(
[
pa.field("title", pa.string()),
pa.field("payload", pa.json_()),
]
)
),
),
]
)
plans = table_module._json_serialization_plans(schema)
assert set(plans) == {"payload", "metadata", "documents"}
metadata_children = plans["metadata"].children
assert metadata_children is not None
assert set(metadata_children) == {"payload"}
documents_item = plans["documents"].item
assert documents_item is not None
assert documents_item.children is not None
assert set(documents_item.children) == {"payload"}
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.asyncio
async def test_add_all_null_json_batch(mem_db_async: AsyncConnection):
python/src/connection.rs
+102 -1
View File
@@ -13,7 +13,11 @@ use crate::{
runtime::future_into_py,
table::Table,
};
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
use arrow::{
datatypes::Schema,
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, ToPyArrow},
};
use lancedb::{
connection::Connection as LanceConnection,
connection::NamespaceClientPushdownOperation,
@@ -100,6 +104,28 @@ fn parse_default_namespace_path(path: Option<Bound<'_, PyAny>>) -> PyResult<Vec<
}
}
/// A view description on its way to Python: name, namespace, query, default
/// database, and the schema as pyarrow renders it.
type PyViewDescription = (String, Vec<String>, String, String, Vec<String>, Py<PyAny>);
/// A view description as a plain tuple, with the schema converted to the
/// pyarrow schema the caller would get from any other lancedb API. The Python
/// layer names the fields; this keeps the binding free of a class that would
/// have to be kept in step with the Rust struct.
fn view_description_to_py(view: lancedb::view::ViewDescription) -> PyResult<PyViewDescription> {
Python::attach(|py| {
let schema = view.schema.to_pyarrow(py)?.unbind();
Ok((
view.name,
view.namespace_path,
view.query,
view.default_database,
view.default_namespace_path,
schema,
))
})
}
#[pymethods]
impl Connection {
fn __repr__(&self) -> String {
@@ -768,6 +794,21 @@ impl Connection {
})
}
pub fn drop_function_async(
self_: PyRef<'_, Self>,
name: String,
version: String,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner
.drop_function_async(name, version)
.await
.infer_error()
.map(|(dropped, job)| (dropped, crate::job::Job::new(job)))
})
}
#[pyo3(signature = (name, value, namespace_path=None))]
pub fn create_secret(
self_: PyRef<'_, Self>,
@@ -847,6 +888,66 @@ impl Connection {
})
}
#[pyo3(signature = (name, query, namespace_path=None))]
pub fn create_view(
self_: PyRef<'_, Self>,
name: String,
query: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
let view = inner
.create_view(name, query, &namespace_path)
.await
.infer_error()?;
view_description_to_py(view)
})
}
#[pyo3(signature = (name, namespace_path=None))]
pub fn describe_view(
self_: PyRef<'_, Self>,
name: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
let view = inner
.describe_view(name, &namespace_path)
.await
.infer_error()?;
view_description_to_py(view)
})
}
#[pyo3(signature = (name, namespace_path=None))]
pub fn drop_view(
self_: PyRef<'_, Self>,
name: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.drop_view(name, &namespace_path).await.infer_error()
})
}
#[pyo3(signature = (namespace_path=None))]
pub fn list_views(
self_: PyRef<'_, Self>,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.list_views(&namespace_path).await.infer_error()
})
}
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
rust/lancedb/Cargo.toml
+4 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.40.0-beta.3"
version = "0.40.0-beta.6"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
@@ -106,6 +106,9 @@ sha2 = "0.10"
[dev-dependencies]
anyhow = "1"
lance-testing = { workspace = true }
# RestAdapter is only used by cfg(test) code. Keep rest-adapter off the
# production remote feature so Python wheels do not resolve its Axum 0.7 stack.
lance-namespace-impls = { workspace = true, features = ["rest-adapter"] }
tempfile = { workspace = true }
random_word = { version = "0.4.3", features = ["en"] }
tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] }
@@ -169,7 +172,6 @@ remote = [
"dep:webbrowser",
"dep:fs4",
"lance-namespace-impls/rest",
"lance-namespace-impls/rest-adapter",
]
# Publish LanceDB's internal metrics (currently object store request counts,
# bytes, latency, errors, and throttles) through the `metrics` crate facade,
rust/lancedb/src/connection.rs
+109 -1
View File
@@ -37,7 +37,11 @@ use crate::remote::{
},
};
use crate::secrets::SecretInfo;
use crate::utils::{validate_secret_component, validate_secret_reference};
use crate::utils::{
validate_namespace, validate_secret_component, validate_secret_reference,
validate_view_reference,
};
use crate::view::ViewDescription;
use lance::io::ObjectStoreParams;
pub use lance_file::version::LanceFileVersion;
#[cfg(feature = "remote")]
@@ -649,6 +653,22 @@ impl Connection {
.await
}
/// Start dropping a Function and return its cleanup job.
///
/// The name is unbound before this returns; the object's content may still be being
/// deleted. Await [`Job::wait`][crate::job::Job::wait] to wait for that to finish. When
/// the server deletes inline, or when nothing was bound, the returned job is already
/// finished and has no id. Local databases return [`Error::NotSupported`].
pub async fn drop_function_async(
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
) -> Result<(bool, crate::job::Job)> {
self.internal
.drop_function_async(name.as_ref(), version.as_ref())
.await
}
/// Create a named Secret in this database.
///
/// Fails if the name is taken, so a create can never silently become a
@@ -731,6 +751,94 @@ impl Connection {
.await
}
/// Create a view: a named query the database plans on every read.
///
/// The query is planned once, here, so one that cannot be planned is
/// refused now rather than at the first read. A view holds no rows, and
/// its readers see its sources as they are at read time.
///
/// There is no replace: a name already taken is an error, and changing a
/// view is a drop followed by a create, each authorized against what it
/// actually touches. Local databases return [`Error::NotSupported`].
///
/// # Example
///
/// ```no_run
/// # async fn view_lifecycle(
/// # connection: &lancedb::Connection,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// let namespace = vec!["analytics".to_string()];
///
/// let view = connection
/// .create_view(
/// "recent_orders",
/// "SELECT id, total FROM orders WHERE total > 100",
/// &namespace,
/// )
/// .await?;
/// println!("{} has {} columns", view.name, view.schema.fields().len());
///
/// // The query comes back as it was recorded, with the defaults its
/// // unqualified names resolve against.
/// let described = connection.describe_view("recent_orders", &namespace).await?;
/// println!("{} in {:?}", described.query, described.default_namespace_path);
///
/// let names = connection.list_views(&namespace).await?;
/// assert!(names.iter().any(|name| name == "recent_orders"));
///
/// // Dropping the view leaves `orders` untouched.
/// connection.drop_view("recent_orders", &namespace).await?;
/// # Ok(())
/// # }
/// ```
pub async fn create_view(
&self,
name: impl AsRef<str>,
query: impl AsRef<str>,
namespace_path: &[String],
) -> Result<ViewDescription> {
validate_view_reference(name.as_ref(), namespace_path)?;
self.internal
.create_view(name.as_ref(), query.as_ref(), namespace_path)
.await
}
/// What this database records about one view: its defining query and the
/// schema that query resolved to.
///
/// The schema is the one recorded at creation. A source altered since then
/// shows up when the view is read, not here. Local databases return
/// [`Error::NotSupported`].
pub async fn describe_view(
&self,
name: impl AsRef<str>,
namespace_path: &[String],
) -> Result<ViewDescription> {
validate_view_reference(name.as_ref(), namespace_path)?;
self.internal
.describe_view(name.as_ref(), namespace_path)
.await
}
/// Drop a view.
///
/// The tables it reads are untouched: a view holds no rows of its own.
/// Local databases return [`Error::NotSupported`].
pub async fn drop_view(&self, name: impl AsRef<str>, namespace_path: &[String]) -> Result<()> {
validate_view_reference(name.as_ref(), namespace_path)?;
self.internal.drop_view(name.as_ref(), namespace_path).await
}
/// The names of the views in one namespace.
///
/// Names only; a definition is query metadata and comes from
/// [`Self::describe_view`]. The client walks all server pages before
/// returning. Local databases return [`Error::NotSupported`].
pub async fn list_views(&self, namespace_path: &[String]) -> Result<Vec<String>> {
validate_namespace(namespace_path)?;
self.internal.list_views(namespace_path).await
}
/// Rename a table in the database.
///
/// This is only supported in LanceDB Cloud.
rust/lancedb/src/database.rs
+48
View File
@@ -32,6 +32,7 @@ use crate::job::Job;
use crate::materialized_view::CreateMaterializedViewRequest;
use crate::secrets::SecretInfo;
use crate::table::{BaseTable, WriteOptions};
use crate::view::ViewDescription;
pub mod listing;
pub mod namespace;
@@ -258,6 +259,12 @@ fn secret_catalog_not_supported<T>() -> Result<T> {
})
}
fn view_ops_not_supported<T>() -> Result<T> {
Err(crate::error::Error::NotSupported {
message: "View operations are not supported by this database".to_string(),
})
}
/// The `Database` trait defines the interface for database implementations.
///
/// A database is responsible for managing tables and their metadata.
@@ -393,6 +400,18 @@ pub trait Database:
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
/// Start dropping a Function and return a handle to the cleanup job.
///
/// Backends without asynchronous cleanup complete the drop before returning an
/// already-finished job.
async fn drop_function_async(
&self,
name: &str,
version: &str,
) -> Result<(bool, crate::job::Job)> {
let dropped = self.drop_function(name, version).await?;
Ok((dropped, crate::job::Job::new_done()))
}
/// Create a named Secret in this database. Fails if the name is taken, so
/// a create can never silently become a rotation.
async fn create_secret(
@@ -431,6 +450,35 @@ pub trait Database:
async fn describe_secret(&self, _name: &str, _namespace_path: &[String]) -> Result<SecretInfo> {
secret_catalog_not_supported()
}
/// Create a view from a defining query. The query is planned once, by the
/// database, so one that cannot be planned is refused now rather than at
/// the first read.
async fn create_view(
&self,
_name: &str,
_query: &str,
_namespace_path: &[String],
) -> Result<ViewDescription> {
view_ops_not_supported()
}
/// What the database records about one view: its defining query and the
/// schema that query resolved to.
async fn describe_view(
&self,
_name: &str,
_namespace_path: &[String],
) -> Result<ViewDescription> {
view_ops_not_supported()
}
/// Drop a view. Its sources are untouched -- a view holds no rows of its
/// own.
async fn drop_view(&self, _name: &str, _namespace_path: &[String]) -> Result<()> {
view_ops_not_supported()
}
/// The names of the views in one namespace.
async fn list_views(&self, _namespace_path: &[String]) -> Result<Vec<String>> {
view_ops_not_supported()
}
/// Open a job by id, returning a handle with its record already
/// populated. Fails with [`crate::Error::JobNotFound`] when the server has
/// no such job.
rust/lancedb/src/lib.rs
+1
View File
@@ -202,6 +202,7 @@ pub mod table;
#[cfg(test)]
pub mod test_utils;
pub mod utils;
pub mod view;
use std::{fmt::Display, str::FromStr};
rust/lancedb/src/materialized_view.rs
+112 -40
View File
@@ -9,6 +9,8 @@
//! query this version cannot maintain reads back as unrefreshable, not as a
//! plain table. Queries, indexes and search work on the view unchanged.
mod grouped;
pub use grouped::IVF_PARTITION;
mod query;
pub mod refresh;
@@ -79,13 +81,21 @@ const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions";
/// produces, which is what lets a query embed its own text.
const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions";
/// The layout this version writes under [`DEFINITION_META_KEY`]:
/// `{"format": 1, "query": "<SQL>"}`, the query as
/// [`MaterializedViewDefinition::to_sql`] renders it. A reader refuses a
/// newer format rather than guess at it. The layout also carries
/// `"kind": "query"`, which readers older than the format number report
/// as an unrefreshable view instead of failing to read the metadata.
pub const DEFINITION_FORMAT: u64 = 1;
/// The newest layout this version reads under [`DEFINITION_META_KEY`]:
/// `{"format": N, "query": "<SQL>"}`, the query as
/// [`MaterializedViewDefinition::to_sql`] renders it. Format 2 is a query
/// with `GROUP BY`; every other query is written as format 1, so a reader
/// that predates grouping reports a grouped view as unrefreshable rather
/// than failing to parse it. A reader refuses a newer format rather than
/// guess at it. The layout also carries `"kind": "query"`, which readers
/// older than the format number report as an unrefreshable view instead of
/// failing to read the metadata.
pub const DEFINITION_FORMAT: u64 = 2;
/// The format `definition` is written in; see [`DEFINITION_FORMAT`].
fn format_of(definition: &MaterializedViewDefinition) -> u64 {
if definition.is_grouped() { 2 } else { 1 }
}
/// Legacy `kind` tag of the structured layout written before
/// [`DEFINITION_FORMAT`] existed; still read, never written.
@@ -240,6 +250,9 @@ pub struct MaterializedViewDefinition {
pub projections: Vec<ViewProjection>,
/// SQL predicate selecting the rows the view holds.
pub filter: Option<String>,
/// `GROUP BY` expressions. A grouped view holds one row per group and is
/// recomputed in full whenever its source changes.
pub group_by: Vec<String>,
/// Cap on the number of rows the view holds, in materialization order.
pub limit: Option<u64>,
}
@@ -250,7 +263,7 @@ impl MaterializedViewDefinition {
/// ```sql
/// SELECT <column | expr AS name | *>, ...
/// FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
/// [WHERE predicate] [LIMIT n]
/// [WHERE predicate] [GROUP BY expr, ...] [LIMIT n]
/// ```
///
/// A Function in `FROM` position yields one row per element it returns;
@@ -283,6 +296,11 @@ impl MaterializedViewDefinition {
definition_to_metadata(self)
}
/// Whether the query has a `GROUP BY`.
pub fn is_grouped(&self) -> bool {
!self.group_by.is_empty()
}
/// Whether the query is `SELECT *`.
pub fn selects_star(&self) -> bool {
matches!(self.projections.as_slice(), [p] if *p == ViewProjection::star())
@@ -359,7 +377,7 @@ struct LegacyDefinition {
pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result<String> {
Ok(serde_json::json!({
"kind": "query",
"format": DEFINITION_FORMAT,
"format": format_of(definition),
"query": definition.to_sql(),
})
.to_string())
@@ -389,6 +407,13 @@ pub fn read_definition(metadata: &HashMap<String, String>) -> Result<Option<Stor
return Err(unreadable(&"missing query"));
};
let definition = MaterializedViewDefinition::from_sql(sql).map_err(|e| unreadable(&e))?;
// No correct writer tags a query below the format its shape needs.
if format < format_of(&definition) {
return Err(unreadable(&format!(
"format {format} cannot carry this query; it needs {}",
format_of(&definition)
)));
}
return Ok(Some(StoredDefinition::Query(definition)));
}
let kind = value
@@ -415,6 +440,7 @@ pub fn read_definition(metadata: &HashMap<String, String>) -> Result<Option<Stor
lateral: None,
projections: legacy.projections,
filter: legacy.filter,
group_by: Vec::new(),
limit: legacy.limit,
})))
}
@@ -476,6 +502,9 @@ pub(crate) fn plan(
},
err => err,
})?;
if definition.is_grouped() {
return grouped::plan(source_schema, definition, filter);
}
// Projections are typed against the source, or for an unnested view
// against the source with the list column replaced by its element
// under the alias, where `c.chunk` is an ordinary nested path.
@@ -616,34 +645,7 @@ pub(crate) fn plan(
}
if let Some(filter) = filter.as_deref() {
let expr = planner
.parse_filter(filter)
.map_err(|e| Error::InvalidInput {
message: format!("invalid view filter: {e}"),
})?;
ensure_immutable(&expr, |message| Error::InvalidInput {
message: format!("invalid view filter: {message}"),
})?;
let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput {
message: format!("invalid view filter: {message}"),
})?;
// A committed filter has to be usable as a predicate.
let read_schema = project_schema(&source_schema, &filter_inputs);
let data_type = Planner::new(read_schema.clone())
.create_physical_expr(&expr)
.map_err(|e| Error::InvalidInput {
message: format!("invalid view filter: {e}"),
})?
.data_type(read_schema.as_ref())
.map_err(|e| Error::InvalidInput {
message: format!("invalid view filter: {e}"),
})?;
if data_type != DataType::Boolean {
return Err(Error::InvalidInput {
message: format!("view filter must be a boolean predicate, not {data_type}"),
});
}
inputs.extend(filter_inputs);
inputs.extend(plan_filter(&source_schema, filter)?);
}
let definition = MaterializedViewDefinition {
@@ -655,6 +657,7 @@ pub(crate) fn plan(
.map(|(output, expression)| ViewProjection { output, expression })
.collect(),
filter,
group_by: Vec::new(),
limit,
};
let mut inputs: Vec<String> = inputs
@@ -671,6 +674,39 @@ pub(crate) fn plan(
})
}
/// Check that `filter` is an immutable boolean predicate over `source_schema`,
/// returning the columns it reads.
pub(crate) fn plan_filter(source_schema: &SchemaRef, filter: &str) -> Result<Vec<String>> {
let expr = Planner::new(source_schema.clone())
.parse_filter(filter)
.map_err(|e| Error::InvalidInput {
message: format!("invalid view filter: {e}"),
})?;
ensure_immutable(&expr, |message| Error::InvalidInput {
message: format!("invalid view filter: {message}"),
})?;
let filter_inputs = resolve_inputs(source_schema, &expr, |message| Error::InvalidInput {
message: format!("invalid view filter: {message}"),
})?;
// A committed filter has to be usable as a predicate.
let read_schema = project_schema(source_schema, &filter_inputs);
let data_type = Planner::new(read_schema.clone())
.create_physical_expr(&expr)
.map_err(|e| Error::InvalidInput {
message: format!("invalid view filter: {e}"),
})?
.data_type(read_schema.as_ref())
.map_err(|e| Error::InvalidInput {
message: format!("invalid view filter: {e}"),
})?;
if data_type != DataType::Boolean {
return Err(Error::InvalidInput {
message: format!("view filter must be a boolean predicate, not {data_type}"),
});
}
Ok(filter_inputs)
}
/// The schema a projection over an unnested view is planned against: the
/// source's, with the list column replaced by its element type under the
/// alias.
@@ -1046,6 +1082,15 @@ impl PreparedDeclaration {
/// read: the column the view projects it to, if any, otherwise an
/// internal projection added here, named by [`input_column_name`].
pub fn input_column(&mut self, source_column: &str) -> Result<String> {
// A grouped view's rows are groups; no source row carries a value into one.
if self.definition.is_grouped() {
return Err(Error::InvalidInput {
message: format!(
"a computed column on a grouped view cannot read source column \
'{source_column}'; declare it on a view over the grouped view"
),
});
}
if let Some(output) = self.lineage.get(source_column).and_then(|o| o.first()) {
return Ok(output.clone());
}
@@ -1357,6 +1402,7 @@ pub async fn prepare_declaration(
.collect(),
},
filter: filter.map(str::to_string),
group_by: Vec::new(),
limit,
};
prepare_definition(source, definition).await
@@ -1383,6 +1429,26 @@ pub async fn prepare_declaration(
/// # Ok(())
/// # }
/// ```
///
/// A grouped view holds one row per group and is recomputed in full on
/// every refresh after a source change:
///
/// ```
/// # #![recursion_limit = "256"]
/// use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition};
///
/// # async fn declare(events: &lancedb::Table) -> Result<(), Box<dyn std::error::Error>> {
/// let definition = MaterializedViewDefinition::from_sql(
/// "SELECT kind, count(*) AS n, array_agg(id) AS ids FROM events GROUP BY kind",
/// )?;
/// let view = prepare_definition(events, definition)
/// .await?
/// .create("events_by_kind")
/// .await?;
/// view.refresh().execute().await?;
/// # Ok(())
/// # }
/// ```
pub async fn prepare_definition(
source: &Table,
definition: MaterializedViewDefinition,
@@ -1532,6 +1598,9 @@ async fn prepare_with(
lineage,
..
} = plan(source_schema.clone(), &definition, staging.as_ref())?;
if definition.is_grouped() {
grouped::check(native.dataset.get().await?.as_ref(), &definition).await?;
}
// What later projections (`input_column`) are planned against: for an
// unnested view the flattened schema, where the alias is a column.
let planning_schema = match physical_unnest(&definition, staging.as_ref())? {
@@ -2067,6 +2136,7 @@ mod tests {
},
],
filter: Some("age >= 18".into()),
group_by: Vec::new(),
limit: Some(10),
lateral: None,
}
@@ -3110,6 +3180,7 @@ mod tests {
expression: "name".to_string(),
}],
filter: None,
group_by: Vec::new(),
limit: None,
}
}
@@ -3187,8 +3258,8 @@ mod tests {
#[test]
fn a_newer_format_is_reported_not_guessed() {
assert_eq!(
read(r#"{"format":2,"query":"SELECT name FROM people"}"#).unwrap(),
Some(StoredDefinition::Newer { format: "2".into() })
read(r#"{"format":3,"query":"SELECT name FROM people"}"#).unwrap(),
Some(StoredDefinition::Newer { format: "3".into() })
);
assert_eq!(
read(r#"{"kind":"join"}"#).unwrap(),
@@ -3201,6 +3272,7 @@ mod tests {
r#"{"format":"one"}"#,
r#"{"format":1}"#,
r#"{"format":1,"query":"SELECT name FROM people GROUP BY name"}"#,
r#"{"format":2,"query":"SELECT name FROM people ORDER BY name"}"#,
] {
assert!(read(stored).is_err(), "{stored}");
}
@@ -3399,7 +3471,7 @@ mod tests {
// The stored definition is the query alone; bindings live beside it.
let stored: serde_json::Value =
serde_json::from_str(&schema.metadata()[DEFINITION_META_KEY]).unwrap();
assert_eq!(stored["format"], DEFINITION_FORMAT);
assert_eq!(stored["format"], 1);
assert_eq!(view.definition().projections.len(), 2);
assert_eq!(view.table().count_rows(None).await.unwrap(), 0);
assert_eq!(conn.open_materialized_view("v").await.unwrap().name(), "v");
rust/lancedb/src/materialized_view/grouped.rs
+527
View File
@@ -0,0 +1,527 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Views with `GROUP BY`: one row per group, planned and executed by
//! DataFusion, and recomputed in full whenever the source changes, since a
//! group spans fragments.
use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use arrow_array::cast::AsArray;
use arrow_array::{Array, FixedSizeListArray, UInt32Array};
use arrow_buffer::NullBuffer;
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef};
use datafusion::catalog::default_table_source::provider_as_source;
use datafusion::datasource::empty::EmptyTable;
use datafusion::execution::FunctionRegistry;
use datafusion::execution::SessionStateBuilder;
use datafusion::execution::context::SessionState;
use datafusion::physical_plan::SendableRecordBatchStream;
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::prelude::SessionContext;
use datafusion_common::DataFusionError;
use datafusion_common::TableReference;
use datafusion_common::config::ConfigOptions;
use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
use datafusion_expr::planner::{ContextProvider, ExprPlanner};
use datafusion_expr::{
AggregateUDF, ColumnarValue, Expr, HigherOrderUDF, LogicalPlan, ScalarFunctionArgs, ScalarUDF,
ScalarUDFImpl, Signature, TableSource, Volatility, WindowUDF,
};
use datafusion_sql::planner::SqlToRel;
use datafusion_sql::sqlparser::dialect::GenericDialect;
use datafusion_sql::sqlparser::parser::Parser;
use futures::StreamExt;
use lance::Dataset;
use lance::index::{DatasetIndexExt, DatasetIndexInternalExt};
use lance_core::ROW_ID;
use lance_datafusion::exec::SessionContextExt;
use lance_index::metrics::NoOpMetricsCollector;
use lance_index::vector::ivf::{IvfTransformer, new_ivf_transformer};
use lance_linalg::distance::DistanceType;
use lance_linalg::kernels::normalize_fsl;
use uuid::Uuid;
use super::refresh::to_view_batch;
use super::{
MaterializedViewDefinition, Planned, SOURCE_ROW_ID_COLUMN, ViewProjection, ensure_immutable,
plan_filter, query, without_declarations,
};
use crate::{Error, Result};
/// The name the source is planned under; never visible to the user.
const SOURCE: &str = "__source";
/// `ivf_partition(column)`: the partition of the IVF index on `column` that
/// each vector falls in, assigned by that index's centroids and distance
/// type. Bound per refresh, so a retrained index regroups the view.
pub const IVF_PARTITION: &str = "ivf_partition";
/// The index `ivf_partition(column)` assigns by.
#[derive(Debug, Clone)]
struct IvfBinding {
index: Uuid,
transformer: Arc<IvfTransformer>,
/// A cosine index assigns L2 over unit vectors, as lance's index path does.
normalize: bool,
}
/// `bindings` maps a source column to its index; planning runs unbound.
#[derive(Debug)]
struct IvfPartition {
signature: Signature,
bindings: HashMap<String, IvfBinding>,
}
impl PartialEq for IvfPartition {
fn eq(&self, other: &Self) -> bool {
self.bindings.len() == other.bindings.len()
&& self.bindings.iter().all(|(column, binding)| {
other
.bindings
.get(column)
.is_some_and(|b| b.index == binding.index)
})
}
}
impl Eq for IvfPartition {}
impl std::hash::Hash for IvfPartition {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
let mut bound: Vec<_> = self.bindings.iter().map(|(c, b)| (c, b.index)).collect();
bound.sort();
bound.hash(state);
}
}
impl ScalarUDFImpl for IvfPartition {
fn name(&self) -> &str {
IVF_PARTITION
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, arg_types: &[DataType]) -> datafusion_common::Result<DataType> {
// Float vectors under L2/dot/cosine, byte vectors under Hamming; the
// binding checks the index's metric against the element type.
match arg_types {
[DataType::FixedSizeList(element, _)]
if element.data_type().is_floating() || *element.data_type() == DataType::UInt8 =>
{
Ok(DataType::UInt32)
}
other => datafusion_common::plan_err!(
"{IVF_PARTITION} takes one vector column, not {other:?}"
),
}
}
fn invoke_with_args(
&self,
args: ScalarFunctionArgs,
) -> datafusion_common::Result<ColumnarValue> {
// The argument is a bare column (checked at planning), so its field
// names the column the binding was made for.
let column = args.arg_fields[0].name();
let Some(binding) = self.bindings.get(column) else {
return datafusion_common::exec_err!("{IVF_PARTITION}({column}) has no index bound");
};
let vectors = args.args[0].to_array(args.number_rows)?;
let vectors = vectors.as_fixed_size_list();
let normalized;
let vectors = if binding.normalize {
normalized =
normalize_fsl(vectors).map_err(|e| DataFusionError::External(Box::new(e)))?;
&normalized
} else {
vectors
};
let partitions = binding
.transformer
.compute_partitions(vectors)
.map_err(|e| DataFusionError::External(Box::new(e)))?;
// A null vector falls in no partition, and neither does one the
// assigner could not place (a zero vector under cosine): a NULL bucket
// is honest, membership in bucket 0 is not.
let partitions = UInt32Array::new(
partitions.values().clone(),
NullBuffer::union(vectors.nulls(), partitions.nulls()),
);
Ok(ColumnarValue::Array(Arc::new(partitions)))
}
}
fn session(bindings: HashMap<String, IvfBinding>) -> SessionState {
let mut state = SessionStateBuilder::new().with_default_features().build();
let ivf = IvfPartition {
signature: Signature::any(1, Volatility::Immutable),
bindings,
};
state
.register_udf(Arc::new(ScalarUDF::new_from_impl(ivf)))
.expect("registering a scalar function cannot fail");
state
}
/// The columns `plan` calls `ivf_partition` on. The argument must be a bare
/// column: its field is how the bound function finds the column's index.
fn ivf_columns(plan: &LogicalPlan) -> Result<Vec<String>> {
let mut columns = Vec::new();
let mut misuse = None;
plan.apply(|node| {
for expr in node.expressions() {
expr.apply(|e| {
if let Expr::ScalarFunction(call) = e
&& call.name() == IVF_PARTITION
{
match call.args.as_slice() {
[Expr::Column(column)] => columns.push(column.name.clone()),
_ => misuse = Some(e.to_string()),
}
}
Ok(TreeNodeRecursion::Continue)
})?;
}
Ok(TreeNodeRecursion::Continue)
})
.map_err(|e| Error::InvalidInput {
message: format!("invalid grouped view: {e}"),
})?;
if let Some(call) = misuse {
return Err(Error::InvalidInput {
message: format!("{IVF_PARTITION} takes a vector column of the source, not `{call}`"),
});
}
columns.sort();
columns.dedup();
Ok(columns)
}
/// Refuse a grouped `definition` whose functions cannot be bound over
/// `source`, such as `ivf_partition` on a column without an IVF index.
pub(super) async fn check(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> {
bind(source, definition).await.map(|_| ())
}
/// Bind every `ivf_partition(column)` in `definition` to the IVF index on
/// that column of `source`.
async fn bind(
source: &Dataset,
definition: &MaterializedViewDefinition,
) -> Result<HashMap<String, IvfBinding>> {
let schema = Arc::new(ArrowSchema::from(source.schema()));
let planned = logical_plan(&session(HashMap::new()), empty_source(&schema), definition)?;
let mut bindings = HashMap::new();
for column in ivf_columns(&planned)? {
bindings.insert(column.clone(), bind_column(source, &column).await?);
}
Ok(bindings)
}
async fn bind_column(source: &Dataset, column: &str) -> Result<IvfBinding> {
let field = source
.schema()
.field(column)
.ok_or_else(|| Error::InvalidInput {
message: format!("{IVF_PARTITION}: the source has no column '{column}'"),
})?;
// One index, whose segments all assign by one model. A segment trained
// on its own fragments carries its own centroids, and grouping every row
// by one segment's model would bucket the other segments' rows wrongly.
let mut found: Option<(String, Uuid, DistanceType, FixedSizeListArray)> = None;
for index in source.load_indices().await?.iter() {
if index.fields != [field.id] {
continue;
}
let Ok(vector) = source
.open_vector_index(column, &index.uuid, &NoOpMetricsCollector)
.await
else {
continue;
};
let Some(centroids) = vector.ivf_model().centroids.clone() else {
continue;
};
let metric = vector.metric_type();
match &found {
None => found = Some((index.name.clone(), index.uuid, metric, centroids)),
Some((name, _, seen_metric, seen)) if *name == index.name => {
if *seen_metric != metric || seen.to_data() != centroids.to_data() {
return Err(Error::InvalidInput {
message: format!(
"{IVF_PARTITION}({column}): the segments of index '{name}' were \
trained separately and assign by different models; rebuild \
the index before grouping by it"
),
});
}
}
Some((name, ..)) => {
return Err(Error::InvalidInput {
message: format!(
"{IVF_PARTITION}({column}) is ambiguous: indices '{name}' and '{}' \
both cover it",
index.name
),
});
}
}
}
let byte_vectors = matches!(
field.data_type(),
DataType::FixedSizeList(element, _) if *element.data_type() == DataType::UInt8
);
let Some((_, uuid, metric, centroids)) = found else {
return Err(Error::InvalidInput {
message: format!(
"{IVF_PARTITION}({column}) needs an IVF vector index on '{column}'; create one first"
),
});
};
// The assigner pairs byte vectors with Hamming and float vectors with the
// rest; an index of the other kind cannot place this column's values.
if byte_vectors != (metric == DistanceType::Hamming) {
return Err(Error::InvalidInput {
message: format!(
"{IVF_PARTITION}({column}): the index assigns by {metric}, which does not \
apply to {}",
field.data_type()
),
});
}
// Lance's index path assigns cosine by L2 over unit vectors.
let normalize = metric == DistanceType::Cosine;
let distance = if normalize { DistanceType::L2 } else { metric };
Ok(IvfBinding {
index: uuid,
transformer: Arc::new(new_ivf_transformer(centroids, distance, vec![])),
normalize,
})
}
fn empty_source(source_schema: &SchemaRef) -> Arc<dyn TableSource> {
let mut fields = source_schema.fields().to_vec();
fields.push(Arc::new(ArrowField::new(ROW_ID, DataType::UInt64, false)));
provider_as_source(Arc::new(EmptyTable::new(Arc::new(ArrowSchema::new(
fields,
)))))
}
/// The query DataFusion runs: the view's projections plus the group's
/// smallest source row id, which stands as the row's provenance. The filter
/// is not here; the lance scan applies it, as for any other view.
fn sql(definition: &MaterializedViewDefinition) -> String {
let items: Vec<String> = definition
.projections
.iter()
.map(|p| format!("{} AS {}", p.expression, query::ident_sql(&p.output)))
.collect();
format!(
"SELECT {}, min({ROW_ID}) AS {ROW_ID} FROM {SOURCE} GROUP BY {}",
items.join(", "),
definition.group_by.join(", ")
)
}
struct Provider<'a> {
state: &'a SessionState,
source: Arc<dyn TableSource>,
}
impl ContextProvider for Provider<'_> {
fn get_table_source(
&self,
name: TableReference,
) -> datafusion_common::Result<Arc<dyn TableSource>> {
if name.schema().is_none() && name.table() == SOURCE {
Ok(self.source.clone())
} else {
datafusion_common::plan_err!("a grouped view reads only its source, not '{name}'")
}
}
fn get_expr_planners(&self) -> &[Arc<dyn ExprPlanner>] {
self.state.expr_planners()
}
fn get_function_meta(&self, name: &str) -> Option<Arc<ScalarUDF>> {
self.state.scalar_functions().get(name).cloned()
}
fn get_higher_order_meta(&self, name: &str) -> Option<Arc<HigherOrderUDF>> {
self.state.higher_order_functions().get(name).cloned()
}
fn get_aggregate_meta(&self, name: &str) -> Option<Arc<AggregateUDF>> {
self.state.aggregate_functions().get(name).cloned()
}
fn get_window_meta(&self, name: &str) -> Option<Arc<WindowUDF>> {
self.state.window_functions().get(name).cloned()
}
fn get_variable_type(&self, _: &[String]) -> Option<DataType> {
None
}
fn options(&self) -> &ConfigOptions {
self.state.config_options()
}
fn udf_names(&self) -> Vec<String> {
self.state.scalar_functions().keys().cloned().collect()
}
fn higher_order_function_names(&self) -> Vec<String> {
self.state
.higher_order_functions()
.keys()
.cloned()
.collect()
}
fn udaf_names(&self) -> Vec<String> {
self.state.aggregate_functions().keys().cloned().collect()
}
fn udwf_names(&self) -> Vec<String> {
self.state.window_functions().keys().cloned().collect()
}
}
fn logical_plan(
state: &SessionState,
source: Arc<dyn TableSource>,
definition: &MaterializedViewDefinition,
) -> Result<LogicalPlan> {
let invalid = |e: &dyn std::fmt::Display| Error::InvalidInput {
message: format!("invalid grouped view: {e}"),
};
let statement = Parser::parse_sql(&GenericDialect {}, &sql(definition))
.map_err(|e| invalid(&e))?
.pop()
.ok_or_else(|| invalid(&"empty query"))?;
SqlToRel::new(&Provider { state, source })
.sql_statement_to_plan(statement)
.map_err(|e| invalid(&e))
}
/// Plan a grouped definition against the source schema. `filter` is the
/// canonical filter, already spelled the way [`super::plan`] stores it.
pub(super) fn plan(
source_schema: SchemaRef,
definition: &MaterializedViewDefinition,
filter: Option<String>,
) -> Result<Planned> {
query::check_grouping(definition)?;
let mut inputs = match &filter {
Some(filter) => plan_filter(&source_schema, filter)?,
None => Vec::new(),
};
let mut projections: Vec<ViewProjection> = Vec::with_capacity(definition.projections.len());
for p in &definition.projections {
if p.output == SOURCE_ROW_ID_COLUMN || p.output == ROW_ID {
return Err(Error::InvalidInput {
message: format!("view column name '{}' is reserved", p.output),
});
}
if projections.iter().any(|seen| seen.output == p.output) {
return Err(Error::ColumnAlreadyExists {
name: p.output.clone(),
});
}
let expression =
query::canonical_expr(&p.expression).map_err(|e| Error::InvalidExpression {
column: p.output.clone(),
message: e.to_string(),
})?;
projections.push(ViewProjection {
output: p.output.clone(),
expression,
});
}
let group_by = definition
.group_by
.iter()
.map(|key| query::canonical_expr(key))
.collect::<Result<Vec<_>>>()?;
let definition = MaterializedViewDefinition {
projections,
filter,
group_by,
..definition.clone()
};
let planned = logical_plan(
&session(HashMap::new()),
empty_source(&source_schema),
&definition,
)?;
ivf_columns(&planned)?;
let mut exprs = Vec::new();
planned
.apply(|node| {
exprs.extend(node.expressions());
Ok(TreeNodeRecursion::Continue)
})
.map_err(|e| Error::InvalidInput {
message: format!("invalid grouped view: {e}"),
})?;
for expr in &exprs {
ensure_immutable(expr, |message| Error::InvalidInput {
message: format!("invalid grouped view: {message}"),
})?;
inputs.extend(
expr.column_refs()
.into_iter()
.map(|c| c.name.clone())
.filter(|name| name != ROW_ID && source_schema.field_with_name(name).is_ok()),
);
}
inputs.sort();
inputs.dedup();
let fields = planned
.schema()
.fields()
.iter()
.filter(|f| f.name() != ROW_ID)
.map(|f| without_declarations(f))
.collect();
Ok(Planned {
definition,
fields,
lineage: HashMap::new(),
inputs,
})
}
/// Every group of `source`, as view rows in `schema`.
pub(super) async fn stream(
source: &Dataset,
definition: &MaterializedViewDefinition,
inputs: &[String],
schema: SchemaRef,
rows_written: Arc<AtomicU64>,
) -> Result<SendableRecordBatchStream> {
let mut scanner = source.scan();
scanner.with_row_id();
if let Some(filter) = &definition.filter {
scanner.filter(filter)?;
}
scanner.project(inputs)?;
let scan: SendableRecordBatchStream = scanner.try_into_stream().await?.into();
let state = session(bind(source, definition).await?);
let ctx = SessionContext::new_with_state(state.clone());
let source = ctx.read_one_shot(scan)?.into_view();
let planned = logical_plan(&state, provider_as_source(source), definition)?;
let groups = ctx
.execute_logical_plan(planned)
.await?
.execute_stream()
.await?;
let out_schema = schema.clone();
let mapped = groups.map(move |batch| {
let batch = to_view_batch(&batch?, &out_schema)?;
rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
Ok(batch)
});
Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped)))
}
rust/lancedb/src/materialized_view/query.rs
+65 -6
View File
@@ -18,8 +18,9 @@
//! query: fail closed, never materialize it at the wrong cardinality.
use datafusion_sql::sqlparser::ast::{
Expr, FunctionArg, FunctionArgExpr, JoinOperator, LimitClause, ObjectName, ObjectNamePart,
Query, SelectItem, SetExpr, Statement, TableFactor, TableFunctionArgs, TableWithJoins, Value,
Expr, FunctionArg, FunctionArgExpr, GroupByExpr, JoinOperator, LimitClause, ObjectName,
ObjectNamePart, Query, SelectItem, SetExpr, Statement, TableFactor, TableFunctionArgs,
TableWithJoins, Value,
};
use datafusion_sql::sqlparser::dialect::GenericDialect;
use datafusion_sql::sqlparser::keywords::{
@@ -40,7 +41,8 @@ fn invalid(message: impl Into<String>) -> Error {
}
const SHAPE: &str = "a materialized view is defined by `SELECT columns FROM table \
[, function(args) AS alias | , UNNEST(column) AS alias] [WHERE predicate] [LIMIT n]`";
[, function(args) AS alias | , UNNEST(column) AS alias] [WHERE predicate] \
[GROUP BY expr, ...] [LIMIT n]`";
/// Whether `name` must be delimited to read back as this identifier: bare,
/// the parser would take it as a keyword, or a different spelling.
@@ -394,14 +396,59 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
Some(_) => return Err(invalid("view limit must be a plain `LIMIT n`")),
};
Ok(MaterializedViewDefinition {
if select.having.is_some() {
return Err(invalid("HAVING is not supported in a view query"));
}
let group_by = match &select.group_by {
GroupByExpr::Expressions(exprs, modifiers)
if modifiers.is_empty()
&& !exprs.iter().any(|e| {
matches!(e, Expr::Rollup(_) | Expr::Cube(_) | Expr::GroupingSets(_))
}) =>
{
exprs.iter().map(|e| e.to_string()).collect()
}
_ => {
return Err(invalid(
"a view groups by a plain `GROUP BY expr, ...` list",
));
}
};
let definition = MaterializedViewDefinition {
source_table,
source_namespace,
lateral,
projections,
filter: select.selection.as_ref().map(|e| e.to_string()),
group_by,
limit,
})
};
check_grouping(&definition)?;
Ok(definition)
}
/// The clauses a grouped view cannot combine with: each would have to be
/// maintained per group rather than per source row.
pub fn check_grouping(definition: &MaterializedViewDefinition) -> Result<()> {
if !definition.is_grouped() {
return Ok(());
}
if definition.lateral.is_some() {
return Err(invalid(
"GROUP BY cannot be combined with a FROM-position item; group in one view \
and expand in a view over it",
));
}
if definition.selects_star() {
return Err(invalid(
"a grouped view selects its keys and aggregates, not `*`",
));
}
if definition.limit.is_some() {
return Err(invalid("LIMIT is not supported together with GROUP BY"));
}
Ok(())
}
/// The canonical query for `definition`; [`parse`] reads it back equal.
@@ -451,6 +498,9 @@ pub fn render(definition: &MaterializedViewDefinition) -> String {
if let Some(filter) = &definition.filter {
sql.push_str(&format!(" WHERE {filter}"));
}
if definition.is_grouped() {
sql.push_str(&format!(" GROUP BY {}", definition.group_by.join(", ")));
}
if let Some(limit) = definition.limit {
sql.push_str(&format!(" LIMIT {limit}"));
}
@@ -489,6 +539,10 @@ mod tests {
"SELECT id, c.text FROM docs, chunk(upper(body)) AS c",
),
("SELECT * FROM `select`.t", "SELECT * FROM `select`.t"),
(
"select k, array_agg(id) as ids, count(*) n from t where x > 1 group by k",
"SELECT k, array_agg(id) AS ids, count(*) AS n FROM t WHERE x > 1 GROUP BY k",
),
(
"SELECT meta.title AS title, id AS id FROM t",
"SELECT meta.title, id FROM t",
@@ -547,7 +601,6 @@ mod tests {
#[test]
fn unsupported_clauses_are_refused() {
for sql in [
"SELECT id FROM t GROUP BY id",
"SELECT id FROM t ORDER BY id",
"SELECT DISTINCT id FROM t",
"SELECT id FROM t LIMIT 5 OFFSET 2",
@@ -567,6 +620,12 @@ mod tests {
"SELECT *, id FROM t",
"WITH q AS (SELECT 1) SELECT id FROM t",
"SELECT id FROM t HAVING id > 1",
"SELECT k, count(*) AS n FROM t GROUP BY k HAVING count(*) > 1",
"SELECT k FROM t GROUP BY ALL",
"SELECT k FROM t GROUP BY ROLLUP (k)",
"SELECT * FROM t GROUP BY k",
"SELECT k FROM t GROUP BY k LIMIT 5",
"SELECT k, c.x FROM t, UNNEST(xs) AS c GROUP BY k",
] {
assert!(parse(sql).is_err(), "{sql}");
}
rust/lancedb/src/materialized_view/refresh.rs
+503 -20
View File
@@ -297,6 +297,22 @@ pub(crate) async fn execute_refresh(
});
}
// A group spans fragments: there is no per-fragment increment.
if definition.is_grouped() {
return rebuild(
view_native,
&view_ds,
&source_ds,
source_version,
source_ts,
definition,
&inputs,
None,
persist.is_some(),
expected_incarnation,
)
.await;
}
let watermark = watermark.filter(|_| view_intact);
match plan_increment(
&source_ds,
@@ -629,9 +645,17 @@ fn same_meaning(
|| stored.limit != replanned.limit
|| stored.projections.len() != replanned.projections.len()
|| stored.filter.is_some() != replanned.filter.is_some()
|| stored.group_by.len() != replanned.group_by.len()
{
return false;
}
// Lance's planner does not read aggregates; a grouped definition is
// compared in its canonical spelling, which planning produced.
if stored.is_grouped() {
return stored.projections == replanned.projections
&& stored.group_by == replanned.group_by
&& stored.filter == replanned.filter;
}
let schema = match unnest {
None => source_schema.clone(),
Some(unnest) => match super::flattened_schema(source_schema, unnest) {
@@ -1288,6 +1312,9 @@ async fn compute_stream(
schema: SchemaRef,
rows_written: Arc<AtomicU64>,
) -> Result<SendableRecordBatchStream> {
if definition.is_grouped() {
return super::grouped::stream(source, definition, inputs, schema, rows_written).await;
}
let RowScope {
fragments,
created_after,
@@ -1363,31 +1390,42 @@ async fn compute_stream(
None => batch,
Some(expanded) => expanded.apply(&batch)?,
};
let mut columns = Vec::with_capacity(out_schema.fields().len());
for field in out_schema.fields() {
if computed_column_from_field(field).is_some() {
columns.push(new_null_array(field.data_type(), batch.num_rows()));
continue;
}
let name = if field.name() == SOURCE_ROW_ID_COLUMN {
ROW_ID
} else {
field.name()
};
let column = batch.column_by_name(name).ok_or_else(|| {
DataFusionError::Internal(format!(
"view column '{}' is not produced by the view's definition",
field.name()
))
})?;
columns.push(column.clone());
}
let batch = to_view_batch(&batch, &out_schema)?;
rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed);
Ok(RecordBatch::try_new(out_schema.clone(), columns)?)
Ok(batch)
});
Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped)))
}
/// `batch` in the view's physical schema: the row id becomes the
/// provenance column and computed columns are written NULL for their owner
/// to fill.
pub(super) fn to_view_batch(
batch: &RecordBatch,
out_schema: &SchemaRef,
) -> datafusion::common::Result<RecordBatch> {
let mut columns = Vec::with_capacity(out_schema.fields().len());
for field in out_schema.fields() {
if computed_column_from_field(field).is_some() {
columns.push(new_null_array(field.data_type(), batch.num_rows()));
continue;
}
let name = if field.name() == SOURCE_ROW_ID_COLUMN {
ROW_ID
} else {
field.name()
};
let column = batch.column_by_name(name).ok_or_else(|| {
DataFusionError::Internal(format!(
"view column '{}' is not produced by the view's definition",
field.name()
))
})?;
columns.push(column.clone());
}
Ok(RecordBatch::try_new(out_schema.clone(), columns)?)
}
/// The post-scan half of an unnested view's refresh: the scan reads
/// `raw_inputs` plus the row id, and each batch is unnested on the list
/// column, filtered, then projected by expressions typed against
@@ -3641,6 +3679,7 @@ mod tests {
},
],
filter: None,
group_by: Vec::new(),
limit: None,
lateral: None,
};
@@ -3675,6 +3714,7 @@ mod tests {
expression: "x".into(),
}],
filter: None,
group_by: Vec::new(),
limit: None,
lateral: None,
};
@@ -4304,4 +4344,447 @@ mod tests {
3
);
}
async fn grouped_source(conn: &Connection) -> Table {
conn.create_table(
"src",
record_batch!(("k", Int32, [1, 1, 2, 3]), ("x", Int32, [10, 11, 20, 30])).unwrap(),
)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap()
}
async fn declare(source: &Table, name: &str, sql: &str) -> Result<MaterializedView> {
crate::materialized_view::prepare_definition(
source,
MaterializedViewDefinition::from_sql(sql)?,
)
.await?
.create(name)
.await
}
/// Rows of `table` as `columns` rendered to text, sorted.
async fn rows(table: &Table, columns: &[&str]) -> Vec<String> {
let batches = table
.query()
.select(Select::columns(columns))
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
let mut rows = Vec::new();
for batch in &batches {
let formatters: Vec<_> = batch
.columns()
.iter()
.map(|c| {
arrow_cast::display::ArrayFormatter::try_new(c.as_ref(), &Default::default())
.unwrap()
})
.collect();
for row in 0..batch.num_rows() {
let cells: Vec<String> = formatters
.iter()
.map(|f| f.value(row).to_string())
.collect();
rows.push(cells.join(" "));
}
}
rows.sort();
rows
}
#[tokio::test]
async fn a_grouped_view_holds_one_row_per_group() {
let conn = connect("memory://").execute().await.unwrap();
let source = grouped_source(&conn).await;
let view = declare(
&source,
"groups",
"SELECT k, array_agg(x) AS xs, sum(x) AS total FROM src WHERE x < 30 GROUP BY k",
)
.await
.unwrap();
assert_eq!(
view.definition().to_sql(),
"SELECT k, array_agg(x) AS xs, sum(x) AS total FROM src WHERE x < 30 GROUP BY k"
);
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Rebuild);
assert_eq!(result.rows_written, 2);
assert_eq!(rows(view.table(), &["k", "total"]).await, ["1 21", "2 20"]);
let unchanged = view.refresh().execute().await.unwrap();
assert_eq!(unchanged.mode, RefreshMode::NoOp);
// Any source change recomputes every group.
source
.add(record_batch!(("k", Int32, [2]), ("x", Int32, [5])).unwrap())
.execute()
.await
.unwrap();
let result = view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Rebuild);
assert_eq!(rows(view.table(), &["k", "total"]).await, ["1 21", "2 25"]);
// A view over the groups expands them again.
let expanded = declare(
view.table(),
"elements",
"SELECT k, e AS x FROM groups, UNNEST(xs) AS e",
)
.await
.unwrap();
expanded.refresh().execute().await.unwrap();
assert_eq!(
rows(expanded.table(), &["k", "x"]).await,
["1 10", "1 11", "2 20", "2 5"]
);
}
/// Provenance is one source row per group, so the view's row-id keys
/// stay unique.
#[tokio::test]
async fn a_grouped_view_records_one_source_row_per_group() {
let conn = connect("memory://").execute().await.unwrap();
let source = grouped_source(&conn).await;
let view = declare(
&source,
"groups",
"SELECT k, count(*) AS n FROM src GROUP BY k",
)
.await
.unwrap();
view.refresh().execute().await.unwrap();
let ids = rows(view.table(), &[SOURCE_ROW_ID_COLUMN]).await;
assert_eq!(ids.len(), 3);
assert_eq!(ids.iter().collect::<HashSet<_>>().len(), 3, "{ids:?}");
}
#[tokio::test]
async fn a_grouped_view_is_planned_at_declaration() {
let conn = connect("memory://").execute().await.unwrap();
let source = grouped_source(&conn).await;
for sql in [
// x is neither a key nor aggregated
"SELECT k, x FROM src GROUP BY k",
"SELECT k, random() AS r FROM src GROUP BY k",
"SELECT k, count(*) AS n FROM src GROUP BY missing",
"SELECT k, count(*) AS __source_row_id FROM src GROUP BY k",
"SELECT k, count(*) AS k FROM src GROUP BY k",
] {
assert!(declare(&source, "v", sql).await.is_err(), "{sql}");
}
let mut grouped = crate::materialized_view::prepare_definition(
&source,
MaterializedViewDefinition::from_sql("SELECT k, count(*) AS n FROM src GROUP BY k")
.unwrap(),
)
.await
.unwrap();
assert!(grouped.input_column("x").is_err());
}
/// Twenty nonzero vectors in two clusters far apart, ids 0-9 and 10-19.
/// Nonzero so every vector has a cosine direction.
async fn clustered_source(conn: &Connection) -> Table {
use arrow_array::types::Float32Type;
use arrow_array::{FixedSizeListArray, Int32Array};
let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..20).map(|i| {
let base = if i < 10 { 0.0 } else { 100.0 };
Some(vec![Some(base + 1.0 + i as f32 * 0.1), Some(base)])
}),
2,
);
let batch = RecordBatch::try_from_iter(vec![
(
"id",
Arc::new(Int32Array::from_iter_values(0..20)) as arrow_array::ArrayRef,
),
("vec", Arc::new(vectors) as arrow_array::ArrayRef),
])
.unwrap();
conn.create_table("images", batch)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap()
}
#[tokio::test]
async fn ivf_partition_groups_by_the_index_on_the_column() {
use crate::index::vector::IvfFlatIndexBuilder;
const BUCKETS: &str = "SELECT ivf_partition(vec) AS bucket, count(*) AS n, \
min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(vec)";
let conn = connect("memory://").execute().await.unwrap();
let source = clustered_source(&conn).await;
let unindexed = declare(&source, "buckets", BUCKETS).await.unwrap_err();
assert!(
unindexed.to_string().contains("IVF vector index"),
"{unindexed}"
);
source
.create_index(
&["vec"],
Index::IvfFlat(IvfFlatIndexBuilder::default().num_partitions(2)),
)
.execute()
.await
.unwrap();
let view = declare(&source, "buckets", BUCKETS).await.unwrap();
view.refresh().execute().await.unwrap();
assert_eq!(
rows(view.table(), &["n", "lo", "hi"]).await,
["10 0 9", "10 10 19"]
);
let buckets = rows(view.table(), &["bucket"]).await;
assert_eq!(buckets, ["0", "1"]);
}
#[tokio::test]
async fn ivf_partition_takes_a_column() {
let conn = connect("memory://").execute().await.unwrap();
let source = clustered_source(&conn).await;
for sql in [
"SELECT ivf_partition(id) AS b, count(*) AS n FROM images GROUP BY ivf_partition(id)",
"SELECT ivf_partition(vec, vec) AS b, count(*) AS n FROM images \
GROUP BY ivf_partition(vec, vec)",
] {
assert!(declare(&source, "v", sql).await.is_err(), "{sql}");
}
}
/// A cosine index assigns by L2 over unit vectors, as lance's own index
/// path does; forwarding cosine to the assigner panics.
#[tokio::test]
async fn ivf_partition_supports_cosine_indices() {
use crate::index::vector::IvfFlatIndexBuilder;
const BUCKETS: &str = "SELECT ivf_partition(vec) AS bucket, count(*) AS n, \
min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(vec)";
let conn = connect("memory://").execute().await.unwrap();
let source = clustered_source(&conn).await;
source
.create_index(
&["vec"],
Index::IvfFlat(
IvfFlatIndexBuilder::default()
.num_partitions(2)
.distance_type(crate::DistanceType::Cosine),
),
)
.execute()
.await
.unwrap();
let view = declare(&source, "cosine_buckets", BUCKETS).await.unwrap();
view.refresh().execute().await.unwrap();
assert_eq!(
rows(view.table(), &["n", "lo", "hi"]).await,
["10 0 9", "10 10 19"]
);
}
/// Segments of one index trained separately carry different centroids;
/// grouping by one of them would bucket the other segments' rows wrongly.
#[tokio::test]
async fn ivf_partition_refuses_segments_with_different_models() {
use arrow_array::types::Float32Type;
use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array};
use lance::index::DatasetIndexExt;
use lance::index::vector::VectorIndexParams;
use lance_index::IndexType;
let conn = connect("memory://").execute().await.unwrap();
let source = clustered_source(&conn).await;
// A second fragment far from the first, so each segment trains its own centroids.
let far = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
(0..20).map(|i| Some(vec![Some(-500.0 - i as f32), Some(-500.0)])),
2,
);
source
.add(
RecordBatch::try_from_iter(vec![
(
"id",
Arc::new(Int32Array::from_iter_values(20..40)) as ArrayRef,
),
("vec", Arc::new(far) as ArrayRef),
])
.unwrap(),
)
.execute()
.await
.unwrap();
let dataset = source.as_native().unwrap().dataset.get().await.unwrap();
let mut dataset = dataset.as_ref().clone();
let params = VectorIndexParams::ivf_flat(2, lance_linalg::distance::DistanceType::L2);
let mut segments = Vec::new();
for fragment in dataset.get_fragments() {
let segment = dataset
.create_index_builder(&["vec"], IndexType::Vector, &params)
.name("shared".to_string())
.fragments(vec![fragment.id() as u32])
.execute_uncommitted()
.await
.unwrap();
segments.push(segment);
}
dataset
.commit_existing_index_segments("shared", "vec", segments)
.await
.unwrap();
assert_eq!(dataset.load_indices().await.unwrap().len(), 2);
let source = conn.open_table("images").execute().await.unwrap();
let error = declare(
&source,
"buckets",
"SELECT ivf_partition(vec) AS b, count(*) AS n FROM images GROUP BY ivf_partition(vec)",
)
.await
.unwrap_err();
assert!(error.to_string().contains("trained separately"), "{error}");
}
/// A vector the assigner cannot place has no bucket: it is grouped under
/// NULL, never under partition 0.
#[tokio::test]
async fn ivf_partition_leaves_an_unassignable_vector_unbucketed() {
use crate::index::vector::IvfFlatIndexBuilder;
use arrow_array::types::Float32Type;
use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array};
let conn = connect("memory://").execute().await.unwrap();
let source = clustered_source(&conn).await;
source
.create_index(
&["vec"],
Index::IvfFlat(
IvfFlatIndexBuilder::default()
.num_partitions(2)
.distance_type(crate::DistanceType::Cosine),
),
)
.execute()
.await
.unwrap();
// The zero vector has no direction, so cosine cannot place it.
let zero = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
[Some(vec![Some(0.0), Some(0.0)])],
2,
);
source
.add(
RecordBatch::try_from_iter(vec![
("id", Arc::new(Int32Array::from(vec![20])) as ArrayRef),
("vec", Arc::new(zero) as ArrayRef),
])
.unwrap(),
)
.execute()
.await
.unwrap();
let view = declare(
&source,
"buckets",
"SELECT ivf_partition(vec) AS bucket, count(*) AS n, min(id) AS lo, max(id) AS hi \
FROM images GROUP BY ivf_partition(vec)",
)
.await
.unwrap();
view.refresh().execute().await.unwrap();
// ArrayFormatter renders NULL as the empty string; partition numbers
// are the index's own and not asserted.
let groups = rows(view.table(), &["bucket", "n", "lo", "hi"]).await;
let (unbucketed, bucketed): (Vec<_>, Vec<_>) =
groups.iter().partition(|row| row.starts_with(' '));
assert_eq!(unbucketed, [" 1 20 20"], "{groups:?}");
let mut clusters: Vec<&str> = bucketed
.iter()
.map(|row| row.split_once(' ').unwrap().1)
.collect();
clusters.sort();
assert_eq!(clusters, ["10 0 9", "10 10 19"], "{groups:?}");
}
/// Byte vectors group by a Hamming index, the phash case. The centroids
/// are given, as lance's own tests do: two-means over a handful of
/// hashes can converge with one partition empty.
#[tokio::test]
async fn ivf_partition_groups_byte_vectors_by_hamming() {
use arrow_array::types::UInt8Type;
use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array};
use lance::index::DatasetIndexExt;
use lance::index::vector::VectorIndexParams;
use lance_index::IndexType;
use lance_index::vector::ivf::IvfBuildParams;
let hashes = FixedSizeListArray::from_iter_primitive::<UInt8Type, _, _>(
(0..20u8).map(|i| {
let base = if i < 10 { 0x00 } else { 0xff };
Some(vec![
Some(base ^ (i % 4)),
Some(base),
Some(base),
Some(base),
])
}),
4,
);
let batch = RecordBatch::try_from_iter(vec![
(
"id",
Arc::new(Int32Array::from_iter_values(0..20)) as ArrayRef,
),
("phash", Arc::new(hashes) as ArrayRef),
])
.unwrap();
let conn = connect("memory://").execute().await.unwrap();
let source = conn
.create_table("images", batch)
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let centroids = FixedSizeListArray::from_iter_primitive::<UInt8Type, _, _>(
[Some(vec![Some(0); 4]), Some(vec![Some(0xff); 4])],
4,
);
let params = VectorIndexParams::with_ivf_flat_params(
lance_linalg::distance::DistanceType::Hamming,
IvfBuildParams::try_with_centroids(2, Arc::new(centroids)).unwrap(),
);
let mut dataset = source
.as_native()
.unwrap()
.dataset
.get()
.await
.unwrap()
.as_ref()
.clone();
dataset
.create_index(&["phash"], IndexType::Vector, None, &params, true)
.await
.unwrap();
let source = conn.open_table("images").execute().await.unwrap();
const BUCKETS: &str = "SELECT ivf_partition(phash) AS bucket, count(*) AS n, \
min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(phash)";
let view = declare(&source, "buckets", BUCKETS).await.unwrap();
view.refresh().execute().await.unwrap();
assert_eq!(
rows(view.table(), &["bucket", "n", "lo", "hi"]).await,
["0 10 0 9", "1 10 10 19"]
);
}
}
rust/lancedb/src/remote/db.rs
+368 -4
View File
@@ -14,8 +14,8 @@ use reqwest::header::CONTENT_TYPE;
use lance_namespace::models::{
CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest,
DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest,
ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, JsonArrowSchema,
ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
};
use crate::Error;
@@ -36,6 +36,7 @@ use crate::secrets::SecretBinding;
use crate::secrets::SecretInfo;
use crate::table::BaseTable;
use crate::utils::{reject_relative_segment, validate_table_name};
use crate::view::ViewDescription;
use super::client::{
ClientConfig, HeaderProvider, HttpSend, ID_DELIMITER, RequestResultExt, RestfulLanceDbClient,
@@ -784,6 +785,59 @@ struct RemoteListedSecret {
name: String,
}
/// Define a view from a query. The name and its namespace are the path
/// identifier, so neither appears here.
#[derive(serde::Serialize)]
struct RemoteCreateViewRequest<'a> {
query: &'a str,
}
/// What the service reports about one view. The schema arrives as the
/// namespace spec's JSON encoding, which is what `describe_table` uses too.
#[derive(serde::Deserialize)]
struct RemoteViewDescription {
name: String,
#[serde(default)]
namespace: Vec<String>,
query: String,
default_database: String,
/// A path, like `namespace`: the root is the absent field rather than a
/// spelling of its own.
#[serde(default)]
default_namespace: Vec<String>,
schema: JsonArrowSchema,
}
impl RemoteViewDescription {
fn into_description(self, request_id: String) -> Result<ViewDescription> {
let schema =
lance_namespace::schema::convert_json_arrow_schema(&self.schema).map_err(|source| {
Error::Http {
source: format!("View '{}' has an undecodable schema: {source}", self.name)
.into(),
request_id,
status_code: None,
}
})?;
Ok(ViewDescription {
name: self.name,
namespace_path: self.namespace,
query: self.query,
default_database: self.default_database,
default_namespace_path: self.default_namespace,
schema: Arc::new(schema),
})
}
}
#[derive(serde::Deserialize)]
struct RemoteListViewsResponse {
#[serde(default)]
views: Vec<String>,
#[serde(default)]
page_token: Option<String>,
}
/// Bound on `list_jobs` page walking; a warning is logged when the listing
/// is truncated at this many pages.
const MAX_LIST_JOBS_PAGES: usize = 100;
@@ -1005,6 +1059,10 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
Ok(self.drop_function_async(name, version).await?.0)
}
async fn drop_function_async(&self, name: &str, version: &str) -> Result<(bool, Job)> {
let function_id = build_object_identifier("Function name", name, &[])?;
let req = self
.client
@@ -1014,8 +1072,34 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
}));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?;
Ok(response.dropped)
let status = response.status();
let body = response.text().await.err_to_http(request_id.clone())?;
let dropped: RemoteDropFunctionResponse =
serde_json::from_str(&body).map_err(|source| Error::Http {
source: Box::new(source),
request_id: request_id.clone(),
status_code: Some(status),
})?;
let job = match status {
StatusCode::OK => Job::new_done(),
StatusCode::ACCEPTED => {
let job_id = extract_job_id(&body).ok_or_else(|| Error::Http {
source: "asynchronous Function drop response did not contain a valid job_id"
.into(),
request_id,
status_code: Some(status),
})?;
Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id)))
}
_ => {
return Err(Error::Http {
source: "Function drop must return 200 OK or 202 Accepted".into(),
request_id,
status_code: Some(status),
});
}
};
Ok((dropped.dropped, job))
}
async fn create_secret(
@@ -1092,6 +1176,79 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
response.json().await.err_to_http(request_id)
}
async fn create_view(
&self,
name: &str,
query: &str,
namespace_path: &[String],
) -> Result<ViewDescription> {
let view_id = build_object_identifier("View name", name, namespace_path)?;
let req = self
.client
.post(&format!("/v1/view/{view_id}/create"))
.json(&RemoteCreateViewRequest { query });
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let description: RemoteViewDescription =
response.json().await.err_to_http(request_id.clone())?;
description.into_description(request_id)
}
async fn describe_view(
&self,
name: &str,
namespace_path: &[String],
) -> Result<ViewDescription> {
let view_id = build_object_identifier("View name", name, namespace_path)?;
let req = self.client.post(&format!("/v1/view/{view_id}/describe"));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let description: RemoteViewDescription =
response.json().await.err_to_http(request_id.clone())?;
description.into_description(request_id)
}
async fn drop_view(&self, name: &str, namespace_path: &[String]) -> Result<()> {
let view_id = build_object_identifier("View name", name, namespace_path)?;
let req = self.client.post(&format!("/v1/view/{view_id}/drop"));
let (request_id, response) = self.client.send(req).await?;
self.client.check_response(&request_id, response).await?;
Ok(())
}
async fn list_views(&self, namespace_path: &[String]) -> Result<Vec<String>> {
let namespace_id = build_namespace_identifier(namespace_path)?;
let path = format!("/v1/namespace/{namespace_id}/view/list");
let mut views = Vec::new();
let mut page_token: Option<String> = None;
let mut seen_page_tokens = HashSet::new();
loop {
let mut req = self.client.get(&path);
if let Some(token) = &page_token {
req = req.query(&[("page_token", token)]);
}
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let status = response.status();
let response: RemoteListViewsResponse =
response.json().await.err_to_http(request_id.clone())?;
views.extend(response.views);
let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty())
else {
break;
};
if !seen_page_tokens.insert(next_page_token.clone()) {
return Err(Error::Http {
source: "View listing response repeated a page_token".into(),
request_id,
status_code: Some(status),
});
}
page_token = Some(next_page_token);
}
Ok(views)
}
async fn open_job(&self, job_id: &str) -> Result<Job> {
let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string());
match crate::job::JobHandle::describe(&handle).await {
@@ -1783,6 +1940,53 @@ mod tests {
assert_eq!(job.id(), Some("j1-mv-drop"));
}
#[tokio::test]
async fn test_drop_function_async_returns_job() {
let db = super::RemoteDatabase::new_mock(|request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/function/embed/drop");
http::Response::builder()
.status(202)
.body(serde_json::json!({"dropped": true, "job_id": "j1-fn-drop"}).to_string())
.unwrap()
});
let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap();
assert!(dropped);
assert_eq!(job.id(), Some("j1-fn-drop"));
}
/// An unbound name and an inline deletion both answer `200`: the name is gone and
/// nothing is left to wait for, so the job is already finished.
#[tokio::test]
async fn test_drop_function_async_completed_inline() {
let db = super::RemoteDatabase::new_mock(|_| {
http::Response::builder()
.status(200)
.body(serde_json::json!({"dropped": false}).to_string())
.unwrap()
});
let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap();
assert!(!dropped);
assert_eq!(job.id(), None);
assert_eq!(job.status().await.unwrap(), "finished");
job.wait().await.unwrap();
}
#[tokio::test]
async fn test_drop_function_async_rejects_incomplete_acceptance() {
for body in [
r#"{"dropped":true}"#,
r#"{"dropped":true,"job_id":""}"#,
r#"{"dropped":true,"job_id":null}"#,
] {
let db = super::RemoteDatabase::new_mock(move |_| {
http::Response::builder().status(202).body(body).unwrap()
});
let error = db.drop_function_async("embed", "1").await.err().unwrap();
assert!(error.to_string().contains("valid job_id"));
}
}
#[tokio::test]
async fn test_drop_materialized_view_completed_inline() {
let db = super::RemoteDatabase::new_mock(|request| {
@@ -3721,6 +3925,166 @@ mod tests {
.unwrap();
}
/// A view description carries the schema in the namespace spec's JSON
/// encoding, so the body a test asserts against is the one that encoding
/// produces rather than a hand-written guess at it.
fn view_description_body(name: &str, namespace: &[&str], query: &str) -> String {
let schema = Schema::new(vec![Field::new("id", DataType::Int32, true)]);
serde_json::json!({
"name": name,
"namespace": namespace,
"query": query,
"default_database": "db",
"default_namespace": namespace,
"schema": lance_namespace::schema::arrow_schema_to_json(&schema).unwrap(),
})
.to_string()
}
#[tokio::test]
async fn test_create_view_posts_the_query_and_returns_the_resolved_schema() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/view/analytics$adults/create");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
// The name and its namespace address the view in the path, so the
// body says only what they cannot.
assert_eq!(body, serde_json::json!({"query": "SELECT id FROM people"}));
http::Response::builder()
.status(200)
.body(view_description_body(
"adults",
&["analytics"],
"SELECT id FROM people",
))
.unwrap()
});
let view = conn
.create_view("adults", "SELECT id FROM people", &["analytics".into()])
.await
.unwrap();
assert_eq!(view.name, "adults");
assert_eq!(view.namespace_path, vec!["analytics".to_string()]);
assert_eq!(view.query, "SELECT id FROM people");
assert_eq!(view.default_database, "db");
assert_eq!(view.default_namespace_path, vec!["analytics".to_string()]);
assert_eq!(view.schema.field(0).name(), "id");
}
#[tokio::test]
async fn test_describe_and_drop_address_the_view_in_the_path() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.url().path(), "/v1/view/adults/describe");
assert!(request.body().is_none(), "{:?}", request.body());
http::Response::builder()
.status(200)
.body(view_description_body(
"adults",
&[],
"SELECT id FROM people",
))
.unwrap()
});
let view = conn.describe_view("adults", &[]).await.unwrap();
assert!(view.namespace_path.is_empty());
assert_eq!(view.schema.fields().len(), 1);
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/view/analytics$adults/drop");
assert!(request.body().is_none(), "{:?}", request.body());
http::Response::builder().status(200).body("{}").unwrap()
});
conn.drop_view("adults", &["analytics".into()])
.await
.unwrap();
}
/// A schema the client cannot decode is a broken response, not a view
/// with no columns: reporting it as an error keeps a caller from reading
/// an empty schema as the truth about the view.
#[tokio::test]
async fn test_describe_view_rejects_an_undecodable_schema() {
let conn = Connection::new_with_handler(|_| {
http::Response::builder()
.status(200)
.body(
r#"{"name":"adults","query":"SELECT 1","default_database":"db",
"schema":{"fields":[{"name":"id","type":{"type":"nonesuch"},
"nullable":true}]}}"#,
)
.unwrap()
});
let error = conn.describe_view("adults", &[]).await.unwrap_err();
assert!(error.to_string().contains("undecodable schema"), "{error}");
}
#[tokio::test]
async fn test_list_views_walks_pages() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::GET);
assert_eq!(request.url().path(), "/v1/namespace/analytics/view/list");
let page = request
.url()
.query_pairs()
.find(|(key, _)| key == "page_token")
.map(|(_, value)| value.into_owned());
let body = match page.as_deref() {
None => r#"{"views":[],"page_token":"p2"}"#,
Some("p2") => r#"{"views":["adults"]}"#,
Some(other) => panic!("unexpected page token: {other}"),
};
http::Response::builder().status(200).body(body).unwrap()
});
assert_eq!(
conn.list_views(&["analytics".into()]).await.unwrap(),
vec!["adults".to_string()]
);
}
/// A server that keeps handing back the same token would otherwise spin
/// forever.
#[tokio::test]
async fn test_list_views_rejects_a_repeated_page_token() {
let conn = Connection::new_with_handler(|_| {
http::Response::builder()
.status(200)
.body(r#"{"views":["adults"],"page_token":"same"}"#)
.unwrap()
});
let error = conn.list_views(&[]).await.unwrap_err();
assert!(
error.to_string().contains("repeated a page_token"),
"{error}"
);
}
/// A name carrying the delimiter would split back apart as a different
/// view, so it is refused before it reaches a route.
#[tokio::test]
async fn test_view_names_that_would_resplit_are_refused() {
let conn = Connection::new_with_handler(|_| -> http::Response<String> {
panic!("an invalid identifier must not reach the service")
});
for (name, namespace) in [
("analytics$adults", vec![]),
("adults", vec!["ana$lytics".to_string()]),
("", vec![]),
("..", vec![]),
] {
let error = match conn.describe_view(name, &namespace).await {
Ok(_) => panic!("accepted {name:?} in {namespace:?}"),
Err(error) => error.to_string(),
};
// A view is not a table, so the refusal says so.
assert!(
error.contains("view name") || error.contains("view namespace path segment"),
"{error}"
);
}
}
#[tokio::test]
async fn test_create_function_async_sends_canonical_request_and_decodes_typed_job() {
const REQUEST: &str = include_str!(
rust/lancedb/src/remote/table.rs
+136 -29
View File
@@ -1486,6 +1486,20 @@ fn resolve_arrow_ipc_framing(
})
}
/// An Arrow IPC stream carrying `schema` and no batches.
///
/// The body of an all-null `add_columns`: the server reads the fields to
/// add straight off the schema message, so nothing but the field list is
/// worth sending.
fn write_ipc_schema(schema: &arrow_schema::Schema) -> Result<Vec<u8>> {
let mut body = Vec::new();
{
let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut body, schema)?;
writer.finish()?;
}
Ok(body)
}
/// Strip media-type parameters before matching against the Arrow content types.
fn base_media_type(content_type: &str) -> &str {
match content_type.split_once(';') {
@@ -2130,6 +2144,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})
.collect(),
filter: response.filter,
group_by: Vec::new(),
limit: response.limit,
},
};
@@ -3325,7 +3340,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
"schema evolution",
crate::table::schema_evolution::new_column_names(&transforms),
)?;
match transforms {
let path = format!("/v1/table/{}/add_columns/", self.identifier);
let request = match transforms {
NewColumnTransform::SqlExpressions(expressions) => {
let body = expressions
.into_iter()
@@ -3338,40 +3354,55 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
.collect::<Vec<_>>();
let mut body = serde_json::json!({ "new_columns": body });
self.apply_branch_body(&mut body);
let request = self
.client
.post(&format!("/v1/table/{}/add_columns/", self.identifier))
.json(&body);
let freshness_request = self.snapshot_freshness_headers();
let (request_id, response) = self
.send_with_freshness(request, true, freshness_request)
.await?;
let response = self.check_table_response(&request_id, response).await?;
let body = response.text().await.err_to_http(request_id.clone())?;
if body.trim().is_empty() {
// Backward compatible with old servers
return Ok(AddColumnsResult { version: 0 });
}
let result: AddColumnsResult =
serde_json::from_str(&body).map_err(|e| Error::Http {
source: format!("Failed to parse add_columns response: {}", e).into(),
request_id,
status_code: None,
})?;
self.invalidate_schema_cache();
self.track_write_version(freshness_request, result.version);
Ok(result)
self.client.post(&path).json(&body)
}
// Every field becomes a column that reads null for existing
// rows. The schema goes over the wire as Arrow IPC rather than
// a JSON description of the types: that is what the server
// takes, and it is the only encoding that round-trips a field
// whole, including decimal precision and a timestamp's unit and
// timezone. With no JSON envelope the branch rides the query
// string, as it does for the other binary-bodied endpoints.
NewColumnTransform::AllNulls(schema) => {
let body = write_ipc_schema(schema.as_ref())?;
self.apply_branch_query(
self.client
.post(&path)
.header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE)
.body(body),
)
}
_ => {
return Err(Error::NotSupported {
message: "Only SQL expressions are supported for adding columns".into(),
message: "Only SQL expressions and all-null column schemas are supported \
for adding columns"
.into(),
});
}
};
let freshness_request = self.snapshot_freshness_headers();
let (request_id, response) = self
.send_with_freshness(request, true, freshness_request)
.await?;
let response = self.check_table_response(&request_id, response).await?;
let body = response.text().await.err_to_http(request_id.clone())?;
if body.trim().is_empty() {
// Backward compatible with old servers
return Ok(AddColumnsResult { version: 0 });
}
let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http {
source: format!("Failed to parse add_columns response: {}", e).into(),
request_id,
status_code: None,
})?;
self.invalidate_schema_cache();
self.track_write_version(freshness_request, result.version);
Ok(result)
}
async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
@@ -7913,6 +7944,82 @@ mod tests {
assert_eq!(result.version, if old_server { 0 } else { 43 });
}
/// A pyarrow schema goes over the wire as an Arrow IPC schema under an
/// Arrow content type, not as a JSON description of the types, so a
/// field arrives exactly as sent. The branch has no JSON envelope to
/// ride in and goes on the query string.
#[tokio::test]
async fn test_add_columns_sends_an_arrow_schema_for_all_nulls() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
"/v1/table/my_table/describe/" => simple_describe_response(),
"/v1/table/my_table/add_columns/" => {
assert_eq!(request.method(), "POST");
assert_eq!(
request.headers().get("Content-Type").unwrap(),
ARROW_STREAM_CONTENT_TYPE
);
assert_eq!(
request.url().query_pairs().find(|(k, _)| k == "branch"),
None,
"the main branch sends no branch parameter"
);
// The payload decodes back to the exact schema, types and
// field metadata included.
let body = request.body().unwrap().as_bytes().unwrap();
let reader =
arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(body), None)
.unwrap();
let sent = reader.schema();
assert_eq!(sent.fields().len(), 2);
assert_eq!(sent.field(0).name(), "ts");
assert_eq!(
sent.field(0).data_type(),
&DataType::Timestamp(
arrow_schema::TimeUnit::Nanosecond,
Some("America/New_York".into())
)
);
assert!(sent.field(0).is_nullable());
assert_eq!(sent.field(1).name(), "price");
assert_eq!(sent.field(1).data_type(), &DataType::Decimal128(38, 10));
assert_eq!(
sent.field(1).metadata().get("unit").map(String::as_str),
Some("cents")
);
http::Response::builder()
.status(200)
.body(r#"{"version": 7}"#.to_string())
.unwrap()
}
path => panic!("Unexpected request path: {path}"),
});
let schema = Arc::new(Schema::new(vec![
Field::new(
"ts",
DataType::Timestamp(
arrow_schema::TimeUnit::Nanosecond,
Some("America/New_York".into()),
),
true,
),
Field::new("price", DataType::Decimal128(38, 10), true).with_metadata(
std::collections::HashMap::from([("unit".to_string(), "cents".to_string())]),
),
]));
let result = table
.add_columns()
.transform(NewColumnTransform::AllNulls(schema))
.execute()
.await
.unwrap();
assert_eq!(result.version, 7);
}
/// A declaration is sent as `{name, computed}` for the server to plan; the
/// client never types the expression itself.
#[tokio::test]
rust/lancedb/src/table/computed_columns.rs
+173 -17
View File
@@ -418,23 +418,18 @@ pub(crate) fn ensure_not_function_bound<S: AsRef<str>>(
touched: impl IntoIterator<Item = S>,
) -> Result<()> {
ensure_supported_function_metadata(schema)?;
let mut protected = BTreeSet::new();
for binding in function_bindings(schema)? {
for input in binding.inputs() {
protected.insert(field_root(&input.field_path)?);
}
protected.extend(
binding
.outputs()
.iter()
.map(|output| output.output_name.clone()),
);
protected.extend(
binding
.assignment()
.map(|assignment| assignment.output_name.clone()),
);
}
ensure_not_bound_by(&function_bindings(schema)?, operation, touched)
}
/// [`ensure_not_function_bound`] against an explicit set of bindings, for a
/// caller that is retiring some of the schema's own in the same operation and
/// must be checked against what survives it.
pub(crate) fn ensure_not_bound_by<S: AsRef<str>>(
bindings: &[FunctionBinding],
operation: &str,
touched: impl IntoIterator<Item = S>,
) -> Result<()> {
let protected = protected_roots(bindings)?;
if protected.is_empty() {
return Ok(());
}
@@ -452,6 +447,129 @@ pub(crate) fn ensure_not_function_bound<S: AsRef<str>>(
Ok(())
}
/// Every column `bindings` depend on, inputs taken at their root.
fn protected_roots(bindings: &[FunctionBinding]) -> Result<BTreeSet<String>> {
let mut protected = BTreeSet::new();
for binding in bindings {
for input in binding.inputs() {
protected.insert(field_root(&input.field_path)?);
}
protected.extend(
binding
.outputs()
.iter()
.map(|output| output.output_name.clone()),
);
protected.extend(
binding
.assignment()
.map(|assignment| assignment.output_name.clone()),
);
}
Ok(protected)
}
/// What a drop must do besides removing columns, so that the Function
/// bindings it covers are retired rather than stranded.
#[derive(Debug, Default)]
pub struct FunctionUnbinding {
/// The bindings the drop leaves in place. Later columns in the same
/// operation are checked against these, not against the schema's.
pub retained: Vec<FunctionBinding>,
/// Assignment columns of the retired bindings that the caller did not
/// name. Internal bookkeeping columns, so the drop adds them itself.
pub assignment_columns: Vec<String>,
/// Columns whose `computed_column.*` declaration metadata the unbind
/// commit clears.
pub cleared_columns: Vec<String>,
/// The new value of [`FUNCTION_BINDINGS_META_KEY`]; `None` deletes it.
pub bindings_metadata: Option<String>,
}
impl FunctionUnbinding {
/// True when the drop retires nothing and is an ordinary column drop.
pub fn is_noop(&self) -> bool {
self.cleared_columns.is_empty()
}
/// Refuse `operation` on a path that a binding surviving this drop still
/// reads or writes. The retired ones no longer protect anything.
pub fn ensure_retained_unaffected<S: AsRef<str>>(
&self,
operation: &str,
touched: impl IntoIterator<Item = S>,
) -> Result<()> {
ensure_not_bound_by(&self.retained, operation, touched)
}
}
/// Plan the retirement of every Function binding whose outputs `columns`
/// covers.
///
/// Naming one output of a binding means naming all of them: the outputs of
/// one Function are written by one refresh in one commit, so a binding that
/// kept some of them would have no coherent shape to write. A partial drop is
/// refused and names the siblings that are missing.
pub fn plan_function_unbinding(
schema: &ArrowSchema,
columns: &[&str],
) -> Result<FunctionUnbinding> {
ensure_supported_function_metadata(schema)?;
let named = columns
.iter()
.map(|column| whole_column(column))
.collect::<Result<Vec<Option<String>>>>()?
.into_iter()
.flatten()
.collect::<BTreeSet<String>>();
let mut plan = FunctionUnbinding::default();
for binding in function_bindings(schema)? {
let outputs = binding
.outputs()
.iter()
.map(|output| output.output_name.clone())
.collect::<Vec<_>>();
let Some(dropped) = outputs.iter().find(|name| named.contains(name.as_str())) else {
plan.retained.push(binding);
continue;
};
let missing = outputs
.iter()
.filter(|name| !named.contains(name.as_str()))
.cloned()
.collect::<Vec<_>>();
if !missing.is_empty() {
return Err(Error::InvalidInput {
message: format!(
"dropping Function output '{dropped}' must drop every output of its \
binding; the same request must also name {}",
missing.join(", ")
),
});
}
plan.cleared_columns.extend(outputs);
if let Some(assignment) = binding.assignment() {
plan.cleared_columns.push(assignment.output_name.clone());
if !named.contains(assignment.output_name.as_str()) {
plan.assignment_columns.push(assignment.output_name.clone());
}
}
}
// Re-encoding an untouched set would rewrite bytes a stricter reader
// compares against its own encoding, so leave the key alone when the drop
// retires nothing.
if !plan.is_noop() {
plan.bindings_metadata = if plan.retained.is_empty() {
None
} else {
Some(function_bindings_metadata(&plan.retained)?)
};
}
Ok(plan)
}
/// Read a field's computed-column declaration, if it carries one.
///
/// A field flagged computed but carrying no kind, or a SQL one missing its
@@ -1320,7 +1438,30 @@ pub(crate) fn plan_function_application(
/// Paths are compared at their root: a declaration reading `metadata` is
/// invalidated by a change to `metadata.age` just as surely.
pub(crate) fn ensure_not_an_input(schema: &SchemaRef, paths: &[&str]) -> Result<()> {
ensure_not_an_input_of(schema, paths, &[])
}
/// [`ensure_not_an_input`] where `removed` names columns whose own
/// declarations the same operation deletes. A reader that is going away
/// cannot be stranded by dropping what it reads, so a group drops together.
pub(crate) fn ensure_not_an_input_of(
schema: &SchemaRef,
paths: &[&str],
removed: &[&str],
) -> Result<()> {
// By identity, not spelling: a quoted path names the same column, while
// a path *into* one removes no declaration at all.
let removed = removed
.iter()
.map(|path| whole_column(path))
.collect::<Result<Vec<_>>>()?
.into_iter()
.flatten()
.collect::<BTreeSet<String>>();
for declaration in computed_columns(schema) {
if removed.contains(&declaration.name) {
continue;
}
// The expression, not stored inputs, is the source of truth; an
// expression that no longer parses proves nothing, so refuse.
let inputs = match &declaration.kind {
@@ -1583,6 +1724,21 @@ pub(crate) fn field_root(path: &str) -> Result<String> {
})
}
/// The column a path names when it names the whole column, and `None` when it
/// addresses a field inside one. Retiring a binding takes the whole output;
/// a path into one is left to the ordinary guard to refuse.
fn whole_column(path: &str) -> Result<Option<String>> {
let mut segments = parse_field_path(path)
.map_err(|e| Error::InvalidInput {
message: format!("invalid column path '{path}': {e}"),
})?
.into_iter();
let column = segments.next().ok_or_else(|| Error::InvalidInput {
message: format!("column path '{path}' is empty"),
})?;
Ok(segments.next().is_none().then_some(column))
}
/// [`field_root`], falling back to the text before the first dot for a
/// spelling lance would not resolve.
pub(crate) fn root(path: &str) -> String {
rust/lancedb/src/table/schema_evolution.rs
+423 -24
View File
@@ -9,7 +9,8 @@
//! - [`drop_columns`](execute_drop_columns): Remove columns from the table
use arrow_schema::Schema as ArrowSchema;
use lance::dataset::{ColumnAlteration, NewColumnTransform};
use lance::dataset::transaction::{Operation, Transaction, UpdateMap, UpdateMapEntry};
use lance::dataset::{ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -240,14 +241,96 @@ pub(crate) async fn execute_drop_columns(
table.dataset.ensure_mutable()?;
let mut dataset = (*table.dataset.get().await?).clone();
let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema()));
computed_columns::ensure_not_function_bound(schema.as_ref(), "schema evolution", columns)?;
computed_columns::ensure_not_an_input(&schema, columns)?;
dataset.drop_columns(columns).await?;
let unbinding = computed_columns::plan_function_unbinding(schema.as_ref(), columns)?;
let mut names = columns.iter().map(|c| c.to_string()).collect::<Vec<_>>();
names.extend(unbinding.assignment_columns.iter().cloned());
let dropped = names.iter().map(String::as_str).collect::<Vec<_>>();
computed_columns::ensure_not_bound_by(&unbinding.retained, "schema evolution", &dropped)?;
computed_columns::ensure_not_an_input_of(&schema, &dropped, &dropped)?;
if !unbinding.is_noop() {
// The retirement commits before the projection, so the whole
// projection has to be known valid against this revision first --
// otherwise the binding is gone and the columns are not. Lance plans
// it rather than this repeating the rules: dropping a struct's last
// child removes the struct, and data files that disagree on metadata
// semantics are refused, neither of which an approximation here would
// get right.
dataset.plan_drop_columns(&dropped)?;
dataset = commit_function_unbinding(dataset, &unbinding).await?;
}
dataset.drop_columns(&dropped).await?;
let version = dataset.version().version;
table.dataset.update(dataset);
Ok(DropColumnsResult { version })
}
/// Retire the bindings a drop covers, in the commit before it.
///
/// A binding lives in three places -- the schema-metadata envelope, each
/// output's `computed_column.*` field metadata, and the columns themselves --
/// and readers cross-check the first two, so a state with one of them removed
/// is a table that refuses every write. Clearing both metadata halves in one
/// `UpdateConfig` leaves the outputs as ordinary columns holding their last
/// values: valid on its own, and the drop that follows is then an ordinary
/// drop. Interrupted in between, the columns are still there to be dropped
/// again.
///
/// Not folded into the drop's own `Operation::Project`, though it carries a
/// schema: the transaction proto records `Project` as fields alone, so the
/// metadata edit would survive only in this writer's memory.
async fn commit_function_unbinding(
dataset: Dataset,
unbinding: &computed_columns::FunctionUnbinding,
) -> Result<Dataset> {
let bindings = UpdateMap {
update_entries: vec![UpdateMapEntry {
key: computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(),
value: unbinding.bindings_metadata.clone(),
}],
replace: false,
};
let mut field_metadata_updates = HashMap::new();
for column in &unbinding.cleared_columns {
let field = dataset
.schema()
.field(column)
.ok_or_else(|| Error::InvalidInput {
message: format!("Function output '{column}' does not exist in the table"),
})?;
let cleared = field
.metadata
.keys()
.filter(|key| computed_columns::is_declaration_key(key))
.map(|key| UpdateMapEntry {
key: key.clone(),
value: None,
})
.collect::<Vec<_>>();
field_metadata_updates.insert(
field.id,
UpdateMap {
update_entries: cleared,
replace: false,
},
);
}
let transaction = Transaction::new(
dataset.manifest.version,
Operation::UpdateConfig {
config_updates: None,
table_metadata_updates: None,
schema_metadata_updates: Some(bindings),
field_metadata_updates,
},
None,
);
Ok(CommitBuilder::new(std::sync::Arc::new(dataset))
.execute(transaction)
.await?)
}
/// Internal implementation of the update field metadata logic.
///
/// Merges or replaces per-field metadata, addressing fields by dot-path.
@@ -330,8 +413,9 @@ mod tests {
use crate::query::{ExecutableQuery, QueryBase, Select};
use crate::table::NewColumnTransform;
use crate::table::computed_columns::{
FUNCTION_BINDINGS_META_KEY, ensure_supported_function_metadata, function_bindings,
function_bindings_metadata, function_computed_column_metadata,
FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, FUNCTION_BINDINGS_META_KEY,
ensure_supported_function_metadata, function_bindings, function_bindings_metadata,
function_computed_column_metadata, is_declaration_key,
};
use crate::{Error, Table};
use std::collections::HashMap;
@@ -351,38 +435,71 @@ mod tests {
)
.unwrap();
let table = conn.create_table("bound", batch).execute().await.unwrap();
let binding = FunctionBinding::from_json(include_str!(
stamp_bindings(&table, &[text_features_binding("")]).await;
table
}
/// The fixture binding, its id and output names suffixed so a table can
/// carry more than one.
fn text_features_binding(suffix: &str) -> FunctionBinding {
let raw = include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
))
.unwrap();
);
if suffix.is_empty() {
return FunctionBinding::from_json(raw).unwrap();
}
FunctionBinding::from_json(
&raw.replace("fb_01K3TEXT", &format!("fb_01K3TEXT{suffix}"))
.replace("search_text", &format!("search_text{suffix}"))
.replace("search_token_count", &format!("search_token_count{suffix}")),
)
.unwrap()
}
/// Stamp bindings the way the server does it, since no local path
/// declares one: the envelope on the schema, a declaration marker on
/// every output field.
async fn stamp_bindings(table: &Table, bindings: &[FunctionBinding]) {
let native = table.as_native().unwrap();
let mut dataset = native.dataset.get().await.unwrap().as_ref().clone();
dataset
.update_schema_metadata(vec![(
FUNCTION_BINDINGS_META_KEY.to_string(),
Some(function_bindings_metadata(std::slice::from_ref(&binding)).unwrap()),
Some(function_bindings_metadata(bindings).unwrap()),
)])
.await
.unwrap();
let inputs = ["title".to_string(), "body".to_string()];
let outputs = binding
.outputs()
let outputs = bindings
.iter()
.map(|output| {
(
dataset.schema().field(&output.output_name).unwrap().id as u32,
function_computed_column_metadata(
binding.binding_id(),
output.output_ordinal,
&inputs,
),
)
.flat_map(|binding| {
let assignment = binding.assignment().map(|assignment| {
(
assignment.output_name.clone(),
FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL,
)
});
binding
.outputs()
.iter()
.map(|output| (output.output_name.clone(), output.output_ordinal))
.chain(assignment)
.map(|(name, ordinal)| {
(
dataset.schema().field(&name).unwrap().id as u32,
function_computed_column_metadata(
binding.binding_id(),
ordinal,
&inputs,
),
)
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
dataset.replace_field_metadata(outputs).await.unwrap();
native.dataset.update(dataset);
ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap();
table
}
fn metadata_update(path: &str) -> FieldMetadataUpdate {
@@ -438,6 +555,18 @@ mod tests {
);
}
fn incomplete_group(err: Error) {
assert!(
matches!(&err, Error::InvalidInput { message }
if message.contains("must drop every output of its binding")),
"{err:?}"
);
}
async fn bindings_of(table: &Table) -> Vec<FunctionBinding> {
function_bindings(table.schema().await.unwrap().as_ref()).unwrap()
}
/// Every schema-evolution door refuses a column a binding reads or
/// writes, including a rename onto one.
#[tokio::test]
@@ -445,7 +574,14 @@ mod tests {
let table = bound_table().await;
let version = table.version().await.unwrap();
for column in ["title", "body", "search_text", "search_token_count"] {
bound(table.drop_columns(&[column]).await.unwrap_err());
// A lone drop of an output is refused too, but for its siblings
// rather than for the binding; see the retirement tests below.
let err = table.drop_columns(&[column]).await.unwrap_err();
if column.starts_with("search_") {
incomplete_group(err);
} else {
bound(err);
}
bound(
table
.alter_columns(&[ColumnAlteration::new(column.into()).rename("moved".into())])
@@ -500,7 +636,12 @@ mod tests {
let table = bound_table().await;
let version = table.version().await.unwrap();
for path in ["`title`", "`search_text`", "`title`.nested"] {
bound(table.drop_columns(&[path]).await.unwrap_err());
let err = table.drop_columns(&[path]).await.unwrap_err();
if path == "`search_text`" {
incomplete_group(err);
} else {
bound(err);
}
bound(
table
.alter_columns(&[ColumnAlteration::new(path.into()).set_nullable(false)])
@@ -1038,6 +1179,264 @@ mod tests {
assert_eq!(drop_result.version, v4);
}
// Retiring a Function binding by dropping its outputs (ENT-2669).
/// The whole output set goes, and the binding with it: no envelope, no
/// declaration markers, and the inputs it held are ordinary again.
#[tokio::test]
async fn dropping_every_output_retires_the_binding() {
let table = bound_table().await;
table
.drop_columns(&["search_text", "search_token_count"])
.await
.unwrap();
let schema = table.schema().await.unwrap();
assert!(schema.field_with_name("search_text").is_err());
assert!(schema.field_with_name("search_token_count").is_err());
assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY));
assert!(bindings_of(&table).await.is_empty());
// What the binding used to protect is now ordinary schema.
table.drop_columns(&["title"]).await.unwrap();
table
.update_field_metadata(&[metadata_update("body")])
.await
.unwrap();
ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap();
}
/// One output of a multi-output binding cannot go alone: its siblings are
/// written by the same refresh in the same commit.
#[tokio::test]
async fn dropping_part_of_an_output_group_names_the_missing_siblings() {
let table = bound_table().await;
let version = table.version().await.unwrap();
for (named, missing) in [
("search_text", "search_token_count"),
("search_token_count", "search_text"),
] {
let err = table.drop_columns(&[named]).await.unwrap_err();
let Error::InvalidInput { message } = &err else {
panic!("{err:?}");
};
assert!(
message.contains("must drop every output of its binding")
&& message.contains(missing),
"{message}"
);
}
assert_eq!(table.version().await.unwrap(), version);
assert_eq!(bindings_of(&table).await.len(), 1);
}
/// An input is protected by the binding, not by its own name, so it drops
/// in the request that retires the binding reading it.
#[tokio::test]
async fn an_input_drops_with_the_binding_that_reads_it() {
let table = bound_table().await;
bound(table.drop_columns(&["title"]).await.unwrap_err());
table
.drop_columns(&["title", "body", "search_text", "search_token_count"])
.await
.unwrap();
let schema = table.schema().await.unwrap();
assert_eq!(schema.fields().len(), 1);
assert!(schema.field_with_name("spare").is_ok());
assert!(bindings_of(&table).await.is_empty());
}
/// Lance resolves a quoted spelling to the same field, so it retires the
/// same binding rather than slipping past the group rule.
#[tokio::test]
async fn a_quoted_output_spelling_retires_the_same_binding() {
let table = bound_table().await;
incomplete_group(table.drop_columns(&["`search_text`"]).await.unwrap_err());
table
.drop_columns(&["`search_text`", "`search_token_count`"])
.await
.unwrap();
assert!(bindings_of(&table).await.is_empty());
}
/// The unbind lands before the drop, and on its own: at that version the
/// outputs are still there as plain columns, with no binding and no
/// declaration metadata left pointing at one. A crash in between leaves
/// that state, which every later write must accept.
#[tokio::test]
async fn the_unbind_commit_leaves_a_valid_table_before_the_drop() {
let table = bound_table().await;
let before = table.version().await.unwrap();
table
.drop_columns(&["search_text", "search_token_count"])
.await
.unwrap();
assert_eq!(table.version().await.unwrap(), before + 2);
table.checkout(before + 1).await.unwrap();
let schema = table.schema().await.unwrap();
assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY));
for column in ["search_text", "search_token_count"] {
let field = schema.field_with_name(column).unwrap();
assert!(
!field.metadata().keys().any(|key| is_declaration_key(key)),
"{column}: {:?}",
field.metadata()
);
}
ensure_supported_function_metadata(&schema).unwrap();
assert!(function_bindings(&schema).unwrap().is_empty());
}
/// A table carrying two bindings retires only the one whose outputs the
/// drop names; the other keeps its envelope entry and its protection.
#[tokio::test]
async fn retiring_one_binding_leaves_the_others_standing() {
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(
("title", Utf8, ["a"]),
("body", Utf8, ["b"]),
("search_text", Utf8, ["a b"]),
("search_token_count", Int64, [2]),
("search_text_2", Utf8, ["a b"]),
("search_token_count_2", Int64, [2])
)
.unwrap();
let table = conn.create_table("two", batch).execute().await.unwrap();
stamp_bindings(
&table,
&[text_features_binding(""), text_features_binding("_2")],
)
.await;
table
.drop_columns(&["search_text", "search_token_count"])
.await
.unwrap();
let remaining = bindings_of(&table).await;
assert_eq!(remaining.len(), 1);
assert_eq!(remaining[0].binding_id(), "fb_01K3TEXT_2");
// The survivor still protects the inputs it shared with the retired one.
bound(table.drop_columns(&["title"]).await.unwrap_err());
// And its own outputs are still a group.
incomplete_group(table.drop_columns(&["search_text_2"]).await.unwrap_err());
}
/// A multi-output binding also owns a hidden `__function_assignment_*`
/// column, which the caller never names. Retiring the binding has to take
/// it: leaving it behind orphans a column nothing can fill or explain.
#[tokio::test]
async fn retiring_a_binding_drops_its_assignment_column() {
const ASSIGNMENT: &str = "__function_assignment_fb_01K3TEXT";
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(
("title", Utf8, ["a"]),
("body", Utf8, ["b"]),
("search_text", Utf8, ["a b"]),
("search_token_count", Int64, [2]),
(ASSIGNMENT, Boolean, [Some(true)]),
("spare", Int32, [1])
)
.unwrap();
let table = conn
.create_table("assigned", batch)
.execute()
.await
.unwrap();
let raw = include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
)
.replace(
r#""future_binding""#,
&format!(
r#""assignment": {{"output_name": "{ASSIGNMENT}", "output_field_id": -1}}, "future_binding""#
),
)
// The assignment column is a physical sibling, so it belongs to the
// binding's output schema too -- the server writes it that way.
.replace(
r#"{"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}"#,
&format!(
r#"{{"name": "search_token_count", "nullable": true, "type": {{"type": "int64"}}}}, {{"name": "{ASSIGNMENT}", "nullable": true, "type": {{"type": "bool"}}}}"#
),
);
let binding = FunctionBinding::from_json(&raw).unwrap();
assert!(binding.assignment().is_some(), "fixture must carry one");
stamp_bindings(&table, std::slice::from_ref(&binding)).await;
// The assignment column is the binding's, so it is protected too...
bound(table.drop_columns(&[ASSIGNMENT]).await.unwrap_err());
// ...and goes with the outputs without the caller naming it.
table
.drop_columns(&["search_text", "search_token_count"])
.await
.unwrap();
let schema = table.schema().await.unwrap();
assert!(
schema.field_with_name(ASSIGNMENT).is_err(),
"the assignment column outlived its binding: {:?}",
schema.fields().iter().map(|f| f.name()).collect::<Vec<_>>()
);
assert!(bindings_of(&table).await.is_empty());
ensure_supported_function_metadata(&schema).unwrap();
}
/// An input error must not cost the binding. The retirement commits
/// first, so a projection that could never succeed is refused before it,
/// leaving the table exactly as it was.
#[rstest::rstest]
#[case::unknown_column(
&["search_text", "search_token_count", "nope"],
"does not exist"
)]
#[case::every_column(
&["title", "body", "search_text", "search_token_count", "spare"],
"Cannot drop all columns"
)]
#[case::every_column_quoted(
&["`title`", "`body`", "`search_text`", "`search_token_count`", "`spare`"],
"Cannot drop all columns"
)]
#[tokio::test]
async fn an_invalid_drop_leaves_the_binding_in_place(
#[case] columns: &[&str],
#[case] expected: &str,
) {
let table = bound_table().await;
let version = table.version().await.unwrap();
let err = table.drop_columns(columns).await.unwrap_err();
let Error::InvalidInput { message } = &err else {
panic!("{err:?}");
};
assert!(message.contains(expected), "{message}");
// Read durable state, not this handle: the error path never calls
// `dataset.update`, so the cached handle would report the old version
// even if the unbind had committed.
table.checkout_latest().await.unwrap();
assert_eq!(table.version().await.unwrap(), version);
let schema = table.schema().await.unwrap();
assert!(schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY));
assert_eq!(bindings_of(&table).await.len(), 1);
for column in ["search_text", "search_token_count"] {
assert!(
schema
.field_with_name(column)
.unwrap()
.metadata()
.keys()
.any(|key| is_declaration_key(key)),
"{column} lost its declaration metadata"
);
}
ensure_supported_function_metadata(&schema).unwrap();
}
#[tokio::test]
async fn test_update_field_metadata() {
let conn = connect("memory://").execute().await.unwrap();
rust/lancedb/src/utils/mod.rs
+21
View File
@@ -174,6 +174,27 @@ pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Resul
validate_secret_component("Secret name", name)
}
/// Validate a view name and every segment of the namespace path holding it.
///
/// Worded for a view rather than deferring to [`validate_table_name`]: a view
/// is not a table, and a caller who mistypes one should not be told their
/// table name is invalid. The rule is the same one every object name follows,
/// which is what lets the `$`-joined identifier split back apart.
pub fn validate_view_reference(name: &str, namespace_path: &[String]) -> Result<()> {
for segment in namespace_path {
validate_view_component("view namespace path segment", segment)?;
}
validate_view_component("view name", name)
}
/// Validate one component of a view identifier: a view name, or one segment of
/// the namespace path holding it.
pub fn validate_view_component(what: &str, value: &str) -> Result<()> {
check_object_name(value).map_err(|reason| Error::InvalidInput {
message: format!("invalid {what} '{value}': {reason}"),
})
}
/// Validate all components of a namespace
///
/// Iterates through all namespace components and validates each one.
rust/lancedb/src/view.rs
+44
View File
@@ -0,0 +1,44 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Views: a named query a database stores and plans on every read.
//!
//! A view holds no rows. What it stores is the statement that defines it and
//! the schema that statement resolved to, so a reader sees the sources as they
//! are now rather than as they were when the view was created. That is the
//! whole difference from [`crate::materialized_view`], which holds rows and
//! moves them forward by refresh.
//!
//! The verbs live on [`crate::connection::Connection`]. A view is read through
//! SQL, by name, so there is no `open_view` returning a
//! [`crate::table::Table`]: there would be no rows behind it.
use arrow_schema::SchemaRef;
/// What a database records about one view.
///
/// Returned by [`crate::connection::Connection::describe_view`], and by
/// `create_view` so a caller has the resolved schema without a second call.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ViewDescription {
/// The view's name within its namespace.
pub name: String,
/// The namespace holding the view; empty is the root namespace.
pub namespace_path: Vec<String>,
/// The defining query, as the database stores it.
pub query: String,
/// The database that unqualified table names in `query` resolve against.
/// A view created from a session connected elsewhere can name a database
/// other than its own, so this is part of what the query means rather
/// than a restatement of where the view lives.
pub default_database: String,
/// The namespace path those unqualified names resolve against; empty is
/// the root namespace.
///
/// Recorded with the view because it outlives the session that declared
/// it: a reader that resolved the query against its own default namespace
/// could read a different table than the view was defined over.
pub default_namespace_path: Vec<String>,
/// The schema the defining query resolved to when the view was created.
pub schema: SchemaRef,
}