diff --git a/.bumpversion.toml b/.bumpversion.toml index 314e75779..8387fff7a 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.3" +current_version = "0.40.0-beta.6" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index b80c1b019..5f45ba68c 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -129,12 +129,8 @@ jobs: # link.exe is single-threaded and the long pole on Windows builds. Use # rustc's bundled lld-link instead. CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld - # Fat LTO of the cdylib is single-threaded and the peak-memory step of the - # build. ThinLTO parallelizes it across the runner's cores, at some cost - # to runtime performance on our least performance-sensitive platform. - # Matches what the nodejs Windows builds already do in npm-publish.yml. - CARGO_PROFILE_RELEASE_LTO: thin - CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 16 + # Keep the repository's fat LTO / 1 codegen unit release profile to leave + # headroom below PyPI's file-size limit for Windows wheels. steps: - uses: actions/checkout@v6 with: diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 120000 index 47dc3e3d8..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 681bfd552..19aae6f16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1819,7 +1819,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2937,6 +2937,12 @@ dependencies = [ "sqlparser 0.62.0", ] +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + [[package]] name = "debugid" version = "0.8.0" @@ -3081,7 +3087,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3309,7 +3315,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3499,6 +3505,65 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "foyer-common" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +dependencies = [ + "anyhow", + "bytes", + "cfg-if 1.0.4", + "foyer-tokio", + "mixtrics", + "parking_lot", + "pin-project", + "twox-hash", +] + +[[package]] +name = "foyer-intrusive-collections" +version = "0.10.0-dev" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b" +dependencies = [ + "memoffset", +] + +[[package]] +name = "foyer-memory" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +dependencies = [ + "anyhow", + "asyncband", + "bitflags 2.11.1", + "datasketches", + "equivalent", + "foyer-common", + "foyer-intrusive-collections", + "foyer-tokio", + "futures-util", + "hashbrown 0.17.1", + "itertools 0.15.0", + "mixtrics", + "parking_lot", + "paste", + "pin-project", + "serde", + "tracing", +] + +[[package]] +name = "foyer-tokio" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +dependencies = [ + "tokio", +] + [[package]] name = "frostem" version = "1.20260804.1" @@ -3523,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4004,18 +4069,23 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f521e3beacc809ced36a8aff4a468b1bf61302c8f4dca41c3f21ac0557c22ed" +checksum = "1e471f7bce40936cf2023855c7350e31e11b3f703eb7cb2ce73c9ffc451f87d2" dependencies = [ "arc-swap", "async-trait", "bytes", "dashmap", "fastrand", + "foyer-common", + "foyer-memory", "futures", "hostname", + "io-uring", "itoa", + "libc", + "lru", "prost", "rand 0.9.5", "serde", @@ -4468,7 +4538,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -4810,7 +4880,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5073,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arc-swap", "arrow", @@ -5146,8 +5216,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5169,7 +5239,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5183,7 +5253,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5192,8 +5262,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrayref", "crunchy", @@ -5203,8 +5273,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5241,8 +5311,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5273,8 +5343,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5291,8 +5361,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "proc-macro2", "quote", @@ -5301,8 +5371,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5335,8 +5405,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -5368,8 +5438,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "datafusion", "geo-traits", @@ -5383,8 +5453,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arc-swap", "arrow", @@ -5452,8 +5522,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5475,8 +5545,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5488,6 +5558,7 @@ dependencies = [ "bytes", "chrono", "futures", + "goosefs-sdk", "http 1.5.0", "io-uring", "lance-core", @@ -5498,6 +5569,7 @@ dependencies = [ "object_store 0.14.1", "object_store_opendal", "opendal", + "opendal-service-goosefs", "path_abs", "pin-project", "prost", @@ -5516,8 +5588,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5531,8 +5603,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "async-trait", @@ -5546,8 +5618,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-ipc", @@ -5600,8 +5672,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -5615,8 +5687,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow", "arrow-array", @@ -5656,8 +5728,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "arrow-array", "arrow-schema", @@ -5670,8 +5742,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.6" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.6#93c055dee58553c8b7c816c29892fe32877be3d0" +version = "13.0.0-beta.8" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d" dependencies = [ "frostem", "icu_segmenter", @@ -5684,7 +5756,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.3" +version = "0.40.0-beta.6" dependencies = [ "ahash", "anyhow", @@ -5781,7 +5853,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.3" +version = "0.40.0-beta.6" dependencies = [ "arrow-array", "arrow-buffer", @@ -5806,7 +5878,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.3" +version = "0.40.0-beta.6" dependencies = [ "arc-swap", "arrow", @@ -6207,6 +6279,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.24.6" @@ -6280,6 +6361,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mixtrics" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c46b5adfb7a3ae4996d327a5bdc90e78fec025806dd312bdbe6f07a755e0ec9" +dependencies = [ + "itertools 0.15.0", + "parking_lot", +] + [[package]] name = "moka" version = "0.12.16" @@ -6540,7 +6631,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6669,7 +6760,7 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64 0.22.1", + "base64 0.21.7", "chrono", "getrandom 0.2.17", "http 1.5.0", @@ -8043,8 +8134,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.11.0", "log", "multimap", "petgraph", @@ -8063,7 +8154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8285,7 +8376,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.45", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -8323,7 +8414,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -9123,7 +9214,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9194,7 +9285,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9741,7 +9832,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -9753,7 +9844,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" dependencies = [ - "heck 0.5.0", + "heck 0.4.1", "proc-macro2", "quote", "syn 2.0.117", @@ -9776,7 +9867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -9895,7 +9986,7 @@ dependencies = [ "cfg-if 1.0.4", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -10187,7 +10278,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11208,7 +11299,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7ec2b3538..eb691ac48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.6", default-features = false, "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.6", "tag" = "v13.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/src/java/java.md b/docs/src/java/java.md index ee478561d..391338d1c 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.3 + 0.40.0-beta.6 ``` diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index d46d414d3..75af5102e 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -319,6 +319,38 @@ Creates a new Table and initialize it with new data. *** +### createView() + +```ts +abstract createView( + name, + query, + namespacePath?): Promise +``` + +Create a view: a named query the database plans on every read. + +The query is planned once, at creation, so one that cannot be planned is +rejected now rather than at the first read. A view holds no rows, and its +readers see its sources as they are at read time. + +There is no replace: a name already taken is an error, and changing a +view is a drop followed by a create. + +#### Parameters + +* **name**: `string` + +* **query**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<[`ViewDescription`](../interfaces/ViewDescription.md)> + +*** + ### describeNamespace() ```ts @@ -342,6 +374,27 @@ The namespace's properties *** +### describeView() + +```ts +abstract describeView(name, namespacePath?): Promise +``` + +What this database records about the view named `name`: its defining +query and the schema that query resolved to. + +#### Parameters + +* **name**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<[`ViewDescription`](../interfaces/ViewDescription.md)> + +*** + ### display() ```ts @@ -498,6 +551,28 @@ on the returned job to know when cleanup has finished. *** +### dropView() + +```ts +abstract dropView(name, namespacePath?): Promise +``` + +Drop the view named `name`. + +The tables it reads are untouched: a view holds no rows of its own. + +#### Parameters + +* **name**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<`void`> + +*** + ### isOpen() ```ts @@ -636,6 +711,26 @@ A page of table names and an *** +### listViews() + +```ts +abstract listViews(namespacePath?): Promise +``` + +The names of the views in one namespace. + +Names only; a definition comes from [describeView](Connection.md#describeview). + +#### Parameters + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<`string`[]> + +*** + ### openJob() ```ts diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 6d242a15b..2f49578b1 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -149,6 +149,7 @@ - [UpdateOptions](interfaces/UpdateOptions.md) - [UpdateResult](interfaces/UpdateResult.md) - [Version](interfaces/Version.md) +- [ViewDescription](interfaces/ViewDescription.md) - [WriteExecutionOptions](interfaces/WriteExecutionOptions.md) - [WriteProgress](interfaces/WriteProgress.md) diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md index a1ffb6e2b..68bcb1415 100644 --- a/docs/src/js/interfaces/MaterializedViewDefinition.md +++ b/docs/src/js/interfaces/MaterializedViewDefinition.md @@ -8,8 +8,8 @@ The query that defines a materialized view, as stored: `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] -[WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row -per element it returns. +[WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM` +position yields one row per element it returns. ## Properties diff --git a/docs/src/js/interfaces/ViewDescription.md b/docs/src/js/interfaces/ViewDescription.md new file mode 100644 index 000000000..d53c884b5 --- /dev/null +++ b/docs/src/js/interfaces/ViewDescription.md @@ -0,0 +1,78 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ViewDescription + +# Interface: ViewDescription + +What a database records about one view. + +A view holds no rows: it stores the statement that defines it and the +schema that statement resolved to, so a reader sees its sources as they +are at read time. + +## Properties + +### defaultDatabase + +```ts +defaultDatabase: string; +``` + +The database that unqualified table names in [query](ViewDescription.md#query) resolve against. + +*** + +### defaultNamespacePath + +```ts +defaultNamespacePath: string[]; +``` + +The namespace path those unqualified names resolve against; empty is the +root namespace. + +Recorded with the view because it outlives the session that declared it: +a reader resolving the query against its own default namespace could read +a different table than the view was defined over. + +*** + +### name + +```ts +name: string; +``` + +The view's name within its namespace. + +*** + +### namespacePath + +```ts +namespacePath: string[]; +``` + +The namespace holding the view; empty is the root namespace. + +*** + +### query + +```ts +query: string; +``` + +The defining query, as the database stores it. + +*** + +### schema + +```ts +schema: Schema; +``` + +The schema the defining query resolved to when the view was created. diff --git a/docs/src/python/python.md b/docs/src/python/python.md index acb8b7c2e..3b74f61a6 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -200,6 +200,10 @@ listing a storage directory. ::: lancedb.materialized_view.MaterializedViewDefinition +## Views + +::: lancedb.view.ViewDescription + ## Expressions Type-safe expression builder for filters and projections. Use these instead diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index dda6c2e49..4076ad3bb 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.3 + 0.40.0-beta.6 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index b9d3a2453..ed9cc9e6d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.3 + 0.40.0-beta.6 pom ${project.artifactId} LanceDB Java SDK Parent POM @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.6 + 13.0.0-beta.8 false 2.30.0 1.7 diff --git a/nodejs/CLAUDE.md b/nodejs/CLAUDE.md deleted file mode 120000 index 47dc3e3d8..000000000 --- a/nodejs/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index bbc687a0b..63ed96137 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.3" +version = "0.40.0-beta.6" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts index 352628208..5e5c08745 100644 --- a/nodejs/__test__/materialized_view.test.ts +++ b/nodejs/__test__/materialized_view.test.ts @@ -54,7 +54,7 @@ describe("materialized views", () => { // A newer writer's layout is reported, never guessed at. for (const newer of [ - `{"format":2,"query":${JSON.stringify(query)}}`, + `{"format":3,"query":${JSON.stringify(query)}}`, '{"kind":"select_v3","source_table":"people"}', ]) { expect(() => read(newer)).toThrow(/cannot refresh/); diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 40f4c67db..4c1d21604 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -100,6 +100,67 @@ describe("remote connection", () => { ); }); + it("creates a view and decodes the schema it resolved to", async () => { + await withMockDatabase( + (req, res) => { + expect(req.method).toBe("POST"); + expect(req.url).toBe("/v1/view/analytics$adults/create"); + res.writeHead(200, { "content-type": "application/json" }).end( + JSON.stringify({ + name: "adults", + namespace: ["analytics"], + query: "SELECT name FROM people", + // biome-ignore lint/style/useNamingConvention: the wire field is snake_case + default_database: "db", + // biome-ignore lint/style/useNamingConvention: the wire field is snake_case + default_namespace: ["analytics"], + schema: { + fields: [ + { name: "name", nullable: true, type: { type: "utf8" } }, + ], + }, + }), + ); + }, + async (db) => { + const view = await db.createView("adults", "SELECT name FROM people", [ + "analytics", + ]); + expect(view.name).toBe("adults"); + expect(view.namespacePath).toEqual(["analytics"]); + expect(view.query).toBe("SELECT name FROM people"); + expect(view.defaultDatabase).toBe("db"); + expect(view.defaultNamespacePath).toEqual(["analytics"]); + expect(view.schema.fields.map((f) => f.name)).toEqual(["name"]); + }, + ); + }); + + it("lists and drops views through their own routes", async () => { + await withMockDatabase( + (req, res) => { + expect(req.url).toBe("/v1/namespace/$/view/list"); + res + .writeHead(200, { "content-type": "application/json" }) + .end(JSON.stringify({ views: ["adults"] })); + }, + async (db) => { + expect(await db.listViews()).toEqual(["adults"]); + }, + ); + + await withMockDatabase( + (req, res) => { + expect(req.method).toBe("POST"); + expect(req.url).toBe("/v1/view/adults/drop"); + res.writeHead(200, { "content-type": "application/json" }).end("{}"); + }, + async (db) => { + await db.dropView("adults"); + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index f5a6679cf..9637d60c2 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -31,6 +31,7 @@ import type { ListNamespacesResponse, ListTablesResponse, } from "./native"; +import { ViewDescription, viewDescriptionFromNative } from "./view"; export type { CreateNamespaceResponse, DescribeNamespaceResponse, @@ -377,6 +378,45 @@ export abstract class Connection { namespacePath?: string[], ): Promise; + /** + * Create a view: a named query the database plans on every read. + * + * The query is planned once, at creation, so one that cannot be planned is + * rejected now rather than at the first read. A view holds no rows, and its + * readers see its sources as they are at read time. + * + * There is no replace: a name already taken is an error, and changing a + * view is a drop followed by a create. + */ + abstract createView( + name: string, + query: string, + namespacePath?: string[], + ): Promise; + + /** + * What this database records about the view named `name`: its defining + * query and the schema that query resolved to. + */ + abstract describeView( + name: string, + namespacePath?: string[], + ): Promise; + + /** + * Drop the view named `name`. + * + * The tables it reads are untouched: a view holds no rows of its own. + */ + abstract dropView(name: string, namespacePath?: string[]): Promise; + + /** + * The names of the views in one namespace. + * + * Names only; a definition comes from {@link describeView}. + */ + abstract listViews(namespacePath?: string[]): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -696,6 +736,33 @@ export class LocalConnection extends Connection { ); } + async createView( + name: string, + query: string, + namespacePath?: string[], + ): Promise { + return viewDescriptionFromNative( + await this.inner.createView(name, query, namespacePath ?? []), + ); + } + + async describeView( + name: string, + namespacePath?: string[], + ): Promise { + return viewDescriptionFromNative( + await this.inner.describeView(name, namespacePath ?? []), + ); + } + + async dropView(name: string, namespacePath?: string[]): Promise { + return this.inner.dropView(name, namespacePath ?? []); + } + + async listViews(namespacePath?: string[]): Promise { + return this.inner.listViews(namespacePath ?? []); + } + async listTables( namespacePathOrOptions?: string[] | Partial, options?: Partial, diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index c0a4dfcbd..ed6889232 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -26,6 +26,7 @@ export { MaterializedViewDefinition, MaterializedViewSelect, } from "./materialized_view"; +export { ViewDescription } from "./view"; export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js"; // OpenTelemetry metrics bridge. Only the high-level entry point is public; the diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts index 15ae52274..51f8cfead 100644 --- a/nodejs/lancedb/materialized_view.ts +++ b/nodejs/lancedb/materialized_view.ts @@ -7,14 +7,17 @@ import { Table } from "./table"; /** Schema metadata key holding a materialized view's definition. */ export const DEFINITION_META_KEY = "mv.definition"; -/** The stored layout this version reads: `{"format": 1, "query": ""}`. */ -export const DEFINITION_FORMAT = 1; +/** + * The newest stored layout this version reads: `{"format": N, "query": ""}`, + * format 2 being a query with `GROUP BY`. + */ +export const DEFINITION_FORMAT = 2; /** * The query that defines a materialized view, as stored: * `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] - * [WHERE predicate] [LIMIT n]`. A Function in `FROM` position yields one row - * per element it returns. + * [WHERE predicate] [GROUP BY expr, ...] [LIMIT n]`. A Function in `FROM` + * position yields one row per element it returns. */ export interface MaterializedViewDefinition { /** The defining query, in the canonical spelling the server stores. */ diff --git a/nodejs/lancedb/view.ts b/nodejs/lancedb/view.ts new file mode 100644 index 000000000..5ed848648 --- /dev/null +++ b/nodejs/lancedb/view.ts @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { Schema, tableFromIPC } from "apache-arrow"; +import { ViewDescription as NativeViewDescription } from "./native"; + +/** + * What a database records about one view. + * + * A view holds no rows: it stores the statement that defines it and the + * schema that statement resolved to, so a reader sees its sources as they + * are at read time. + */ +export interface ViewDescription { + /** The view's name within its namespace. */ + name: string; + /** The namespace holding the view; empty is the root namespace. */ + namespacePath: string[]; + /** The defining query, as the database stores it. */ + query: string; + /** The database that unqualified table names in {@link query} resolve against. */ + defaultDatabase: string; + /** + * The namespace path those unqualified names resolve against; empty is the + * root namespace. + * + * Recorded with the view because it outlives the session that declared it: + * a reader resolving the query against its own default namespace could read + * a different table than the view was defined over. + */ + defaultNamespacePath: string[]; + /** The schema the defining query resolved to when the view was created. */ + schema: Schema; +} + +/** Decode the schema the binding hands over as an Arrow IPC file. */ +export function viewDescriptionFromNative( + view: NativeViewDescription, +): ViewDescription { + return { + name: view.name, + namespacePath: view.namespacePath, + query: view.query, + defaultDatabase: view.defaultDatabase, + defaultNamespacePath: view.defaultNamespacePath, + schema: tableFromIPC(view.schema).schema, + }; +} diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index eaf211df2..16a57a94f 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index d51ab1418..b6801e45f 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 1cbab77b7..68b72b6b7 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 572e11db7..f1898b817 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 314c55e28..fc7eb31b6 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 79794ae7f..145a776f2 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 6c0bcd929..8c09fc38a 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 9ccc0885e..b2c51694f 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.3", + "version": "0.40.0-beta.6", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index 0c07ffd50..ee2a9470b 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -55,6 +55,32 @@ pub struct DropNamespaceResponse { pub transaction_id: Option>, } +/// What a database records about one view. +#[napi(object)] +pub struct ViewDescription { + pub name: String, + pub namespace_path: Vec, + pub query: String, + pub default_database: String, + pub default_namespace_path: Vec, + /// The view's schema as an empty Arrow IPC file, the way a table reports + /// its own. + pub schema: Buffer, +} + +impl ViewDescription { + fn from_inner(view: lancedb::view::ViewDescription) -> napi::Result { + Ok(Self { + name: view.name, + namespace_path: view.namespace_path, + query: view.query, + default_database: view.default_database, + default_namespace_path: view.default_namespace_path, + schema: crate::util::schema_to_buffer(&view.schema)?, + }) + } +} + impl Connection { pub(crate) fn inner_new(inner: LanceDBConnection) -> Self { Self { inner: Some(inner) } @@ -368,6 +394,63 @@ impl Connection { .default_error() } + /// Create a view: a named query planned on every read. + #[napi(catch_unwind)] + pub async fn create_view( + &self, + name: String, + query: String, + namespace_path: Option>, + ) -> napi::Result { + let ns = namespace_path.unwrap_or_default(); + let view = self + .get_inner()? + .create_view(&name, &query, &ns) + .await + .default_error()?; + ViewDescription::from_inner(view) + } + + /// What the database records about one view. + #[napi(catch_unwind)] + pub async fn describe_view( + &self, + name: String, + namespace_path: Option>, + ) -> napi::Result { + let ns = namespace_path.unwrap_or_default(); + let view = self + .get_inner()? + .describe_view(&name, &ns) + .await + .default_error()?; + ViewDescription::from_inner(view) + } + + /// Drop a view. The tables it reads are untouched. + #[napi(catch_unwind)] + pub async fn drop_view( + &self, + name: String, + namespace_path: Option>, + ) -> napi::Result<()> { + let ns = namespace_path.unwrap_or_default(); + self.get_inner()? + .drop_view(&name, &ns) + .await + .default_error() + } + + /// The names of the views in one namespace. + #[napi(catch_unwind)] + pub async fn list_views( + &self, + namespace_path: Option>, + ) -> napi::Result> { + let ns = namespace_path.unwrap_or_default(); + self.get_inner()?.list_views(&ns).await.default_error() + } + /// Start dropping a materialized view and return its cleanup job. #[napi(catch_unwind)] pub async fn drop_materialized_view_async( diff --git a/python/CLAUDE.md b/python/CLAUDE.md deleted file mode 120000 index 47dc3e3d8..000000000 --- a/python/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/python/Cargo.toml b/python/Cargo.toml index 9714aff24..40ce3dd84 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.3" +version = "0.40.0-beta.6" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 1f33489db..5c35534a7 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -47,6 +47,7 @@ from .materialized_view import ( MaterializedView, MaterializedViewDefinition, ) +from .view import ViewDescription as ViewDescription from .table import AsyncTable, CompactionOptions, Table from .types import BaseTokenizerType from ._lancedb import Session @@ -582,6 +583,7 @@ __all__ = [ "AsyncMaterializedView", "MaterializedView", "MaterializedViewDefinition", + "ViewDescription", "connect", "connect_async", "tokenize", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 42607f67e..b51535de4 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -153,6 +153,9 @@ class Connection(object): async def get_function(self, name: str, version: str) -> str: ... async def list_functions(self) -> List[str]: ... async def drop_function(self, name: str, version: str) -> bool: ... + async def drop_function_async( + self, name: str, version: str + ) -> Tuple[bool, Job]: ... async def create_secret( self, name: str, value: str, namespace_path: Optional[List[str]] = None ) -> None: ... @@ -168,6 +171,18 @@ class Connection(object): async def describe_secret( self, name: str, namespace_path: Optional[List[str]] = None ) -> Tuple[str, int, int]: ... + async def create_view( + self, name: str, query: str, namespace_path: Optional[List[str]] = None + ) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ... + async def describe_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ... + async def drop_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def list_views( + self, namespace_path: Optional[List[str]] = None + ) -> List[str]: ... async def list_jobs(self) -> List[JobInfo]: ... async def cancel_job(self, job_id: str) -> bool: ... async def execute_query_async( diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index d17e897cf..10d52c63e 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -18,6 +18,7 @@ from typing import ( Literal, Optional, Sequence, + Tuple, Union, ) from uuid import UUID @@ -74,6 +75,7 @@ from .util import ( get_uri_scheme, validate_table_name, ) +from .view import ViewDescription import deprecation @@ -95,6 +97,28 @@ from .namespace_utils import ( ) +def _view_description( + described: Tuple[str, List[str], str, str, List[str], "pa.Schema"], +) -> ViewDescription: + """Name the fields the binding returns positionally.""" + ( + name, + namespace_path, + query, + default_database, + default_namespace_path, + schema, + ) = described + return ViewDescription( + name=name, + query=query, + default_database=default_database, + schema=schema, + namespace_path=namespace_path, + default_namespace_path=default_namespace_path, + ) + + class DBConnection(EnforceOverrides): """An active LanceDB connection interface.""" @@ -828,12 +852,26 @@ class DBConnection(EnforceOverrides): ) def drop_function(self, name: str, *, version: str) -> bool: - """Remove the current Function name binding from the remote catalog. + """Drop a Function name and the object it was bound to. - The requested version must exist in the currently named object. - Object history and existing computed-column references are retained. - Returns True when the name was removed and False when it was absent. - Local connections raise NotImplementedError. + The requested version must exist in the currently named object. Returns + True when the name was removed and False when it was absent. The + object's content is deleted with it, which may finish after this + returns; use :meth:`drop_function_async` to wait for that. Local + connections raise NotImplementedError. + """ + raise NotImplementedError( + "Function catalog operations are not supported for this connection type" + ) + + def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + """Drop a Function name and return its cleanup job. + + The name is unbound before this returns; the object's content may still + be being deleted. Call :meth:`Job.wait` to wait for that to finish. When + the server deletes inline, or when nothing was bound, the returned job + is already finished and has no id. Local connections raise + NotImplementedError. """ raise NotImplementedError( "Function catalog operations are not supported for this connection type" @@ -903,6 +941,68 @@ class DBConnection(EnforceOverrides): "Secret operations are not supported for this connection type" ) + def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """Create a view: a named query the database plans on every read. + + The query is planned once, at creation, so one that cannot be planned + is refused now rather than at the first read. A view holds no rows, and + its readers see its sources as they are at read time. + + There is no replace: a name already taken is an error, and changing a + view is a drop followed by a create. Local connections raise + ``NotImplementedError``. + + >>> import lancedb + >>> db = lancedb.connect("db://my_database") # doctest: +SKIP + >>> view = db.create_view( + ... "adults", "SELECT name FROM people WHERE age >= 18" + ... ) # doctest: +SKIP + >>> view.schema # doctest: +SKIP + name: string + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + + def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """What this database records about a view: its defining query and the + schema that query resolved to. + + The schema is the one recorded at creation; a source altered since then + shows up when the view is read. Local connections raise + ``NotImplementedError``. + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + + def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a view. + + The tables it reads are untouched: a view holds no rows of its own. + Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + + def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + """The names of the views in one namespace. + + Names only; a definition comes from + [describe_view][lancedb.db.DBConnection.describe_view]. Local + connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "View operations are not supported for this connection type" + ) + def open_job(self, job_id: str) -> Job: """Open a server-side job by id, returning a handle with its record already populated. @@ -1683,6 +1783,11 @@ class LanceDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version)) + return dropped, Job(job) + @override def create_secret( self, name: str, value: str, *, namespace_path: Optional[List[str]] = None @@ -1711,6 +1816,30 @@ class LanceDBConnection(DBConnection): ) -> SecretInfo: return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path)) + @override + def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run( + self._conn.create_view(name, query, namespace_path=namespace_path) + ) + + @override + def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run(self._conn.describe_view(name, namespace_path=namespace_path)) + + @override + def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_view(name, namespace_path=namespace_path)) + + @override + def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_views(namespace_path=namespace_path)) + @override def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" @@ -2596,9 +2725,21 @@ class AsyncConnection(object): ] async def drop_function(self, name: str, *, version: str) -> bool: - """Remove the current name binding, retaining the object and its history.""" + """Drop a Function name and the object it was bound to.""" return await self._inner.drop_function(name, version) + async def drop_function_async( + self, name: str, *, version: str + ) -> Tuple[bool, AsyncJob]: + """Drop a Function name and return its cleanup job. + + The name is unbound before this returns; the object's content may still + be being deleted. Await :meth:`AsyncJob.wait` to wait for that to + finish. + """ + dropped, job = await self._inner.drop_function_async(name, version) + return dropped, AsyncJob(job) + async def create_secret( self, name: str, value: str, *, namespace_path: Optional[List[str]] = None ) -> None: @@ -2657,6 +2798,38 @@ class AsyncConnection(object): updated_at_millis=updated_at_millis, ) + async def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """Create a view: a named query the database plans on every read. + + See + [DBConnection.create_view][lancedb.DBConnection.create_view]. + """ + return _view_description( + await self._inner.create_view(name, query, list(namespace_path or [])) + ) + + async def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + """What this database records about a view: query and schema.""" + return _view_description( + await self._inner.describe_view(name, list(namespace_path or [])) + ) + + async def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a view. The tables it reads are untouched.""" + await self._inner.drop_view(name, list(namespace_path or [])) + + async def list_views( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[str]: + """The names of the views in one namespace.""" + return await self._inner.list_views(list(namespace_path or [])) + async def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" return await self._inner.list_jobs() diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py index c487eb044..a836210cb 100644 --- a/python/python/lancedb/materialized_view.py +++ b/python/python/lancedb/materialized_view.py @@ -29,9 +29,10 @@ SelectArg = Union[ ] -DEFINITION_FORMAT = 1 -"""The stored layout this version reads: ``{"format": 1, "query": ""}``. -A ``kind`` key beside it is for readers older than the format number.""" +DEFINITION_FORMAT = 2 +"""The newest stored layout this version reads: ``{"format": N, "query": ""}``, +format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for +readers older than the format number.""" @dataclass @@ -40,7 +41,7 @@ class MaterializedViewDefinition: SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] - [WHERE predicate] [LIMIT n] + [WHERE predicate] [GROUP BY expr, ...] [LIMIT n] A Function in ``FROM`` position yields one row per element it returns. """ diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index c9e857495..962ef66e1 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -16,6 +16,7 @@ from typing import ( List, Optional, Sequence, + Tuple, Union, ) from urllib.parse import urlparse @@ -40,6 +41,7 @@ from ..sql import Query as SqlQuery from ..sql import QueryDescription from ..materialized_view import MaterializedView, SelectArg from ..secrets import EnvVarSecret, SecretInfo +from ..view import ViewDescription if TYPE_CHECKING: from .._lancedb import JobInfo @@ -876,6 +878,11 @@ class RemoteDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def drop_function_async(self, name: str, *, version: str) -> Tuple[bool, Job]: + dropped, job = LOOP.run(self._conn.drop_function_async(name, version=version)) + return dropped, Job(job) + @override def create_secret( self, name: str, value: str, *, namespace_path: Optional[List[str]] = None @@ -904,6 +911,30 @@ class RemoteDBConnection(DBConnection): ) -> None: LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path)) + @override + def create_view( + self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run( + self._conn.create_view(name, query, namespace_path=namespace_path) + ) + + @override + def describe_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> ViewDescription: + return LOOP.run(self._conn.describe_view(name, namespace_path=namespace_path)) + + @override + def drop_view( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_view(name, namespace_path=namespace_path)) + + @override + def list_views(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_views(namespace_path=namespace_path)) + @override def list_jobs(self) -> List["JobInfo"]: """List server-side jobs across the database's tables.""" diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 699fb9925..0e559063f 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -984,6 +984,9 @@ class RemoteTable(Table): self, transforms: Dict[str, str | FunctionApplication] | FunctionApplication + | pa.Field + | List[pa.Field] + | pa.Schema | None = None, *, computed: Dict[str, str] | None = None, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 54bd4c5ea..700263b04 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio import inspect +import json import deprecation import warnings from abc import ABC, abstractmethod @@ -352,12 +353,14 @@ def _into_pyarrow_reader( # convert to list of dict if data is a bunch of LanceModels if isinstance(data[0], LanceModel): - schema = data[0].__class__.to_arrow_schema() + model_schema = data[0].__class__.to_arrow_schema() data = [model_to_dict(d) for d in data] - return pa.Table.from_pylist(data, schema=schema).to_reader() + data = _serialize_json_values(data, schema or model_schema) + return pa.Table.from_pylist(data, schema=model_schema).to_reader() elif isinstance(data[0], pa.RecordBatch): return pa.Table.from_batches(data).to_reader() else: + data = _serialize_json_values(data, schema) return pa.Table.from_pylist(data).to_reader() elif _check_for_pandas(data) and isinstance(data, pd.DataFrame): table = pa.Table.from_pandas(data, preserve_index=False) @@ -705,6 +708,108 @@ def _field_extension_name(field: pa.Field) -> Optional[str]: return extension_name +def _is_json_field(field: pa.Field) -> bool: + return _field_extension_name(field) in ("arrow.json", "lance.json") + + +@dataclass(frozen=True) +class _JsonSerializationPlan: + arrow_field: pa.Field + children: Optional[Dict[str, "_JsonSerializationPlan"]] = None + item: Optional["_JsonSerializationPlan"] = None + + +def _json_serialization_plan(field: pa.Field) -> Optional[_JsonSerializationPlan]: + if _is_json_field(field): + return _JsonSerializationPlan(field) + + if pa.types.is_struct(field.type): + children: Dict[str, _JsonSerializationPlan] = {} + for child_field in field.type: + child_plan = _json_serialization_plan(child_field) + if child_plan is not None: + children[child_field.name] = child_plan + if children: + return _JsonSerializationPlan(field, children=children) + + if _is_list_like(field.type): + item_plan = _json_serialization_plan(field.type.value_field) + if item_plan is not None: + return _JsonSerializationPlan(field, item=item_plan) + + return None + + +def _json_serialization_plans( + schema: pa.Schema, +) -> Dict[str, _JsonSerializationPlan]: + plans: Dict[str, _JsonSerializationPlan] = {} + for field in schema: + plan = _json_serialization_plan(field) + if plan is not None: + plans[field.name] = plan + return plans + + +def _serialize_json_value(value: Any, plan: _JsonSerializationPlan) -> Any: + if value is None or isinstance(value, str): + return value + if _is_json_field(plan.arrow_field): + if isinstance(value, (dict, list)): + return json.dumps(value) + return value + + if plan.children is not None and isinstance(value, dict): + serialized = None + for child_name, child_plan in plan.children.items(): + if child_name not in value: + continue + child_value = _serialize_json_value(value[child_name], child_plan) + if child_value is not value[child_name]: + if serialized is None: + serialized = dict(value) + serialized[child_name] = child_value + return serialized if serialized is not None else value + + if plan.item is not None and isinstance(value, list): + serialized = None + for index, item in enumerate(value): + serialized_item = _serialize_json_value(item, plan.item) + if serialized_item is not item: + if serialized is None: + serialized = list(value) + serialized[index] = serialized_item + return serialized if serialized is not None else value + + return value + + +def _serialize_json_values(data: Any, target_schema: Optional[pa.Schema]) -> Any: + if target_schema is None or not isinstance(data, list): + return data + + plans = _json_serialization_plans(target_schema) + if not plans: + return data + + serialized_rows = [] + for row in data: + if not isinstance(row, dict): + serialized_rows.append(row) + continue + serialized_row = None + for field_name, plan in plans.items(): + if field_name not in row: + continue + value = _serialize_json_value(row[field_name], plan) + if value is not row[field_name]: + if serialized_row is None: + serialized_row = dict(row) + serialized_row[field_name] = value + serialized_rows.append(serialized_row if serialized_row is not None else row) + return serialized_rows + + def _align_field_types( fields: List[pa.Field], target_fields: List[pa.Field], @@ -5819,6 +5924,7 @@ class AsyncTable: """ schema = await self.schema() + data = _serialize_json_values(data, schema) if on_bad_vectors is None: on_bad_vectors = "error" if fill_value is None: diff --git a/python/python/lancedb/view.py b/python/python/lancedb/view.py new file mode 100644 index 000000000..715252fc6 --- /dev/null +++ b/python/python/lancedb/view.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Views: a named query a database stores and plans on every read. + +A view holds no rows. What it stores is the statement that defines it and the +schema that statement resolved to, so a reader sees the sources as they are +now. See ``DBConnection.create_view``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, List + +if TYPE_CHECKING: + import pyarrow as pa + + +@dataclass(frozen=True) +class ViewDescription: + """What a database records about one view.""" + + name: str + """The view's name within its namespace.""" + query: str + """The defining query, as the database stores it.""" + default_database: str + """The database that unqualified table names in ``query`` resolve against.""" + schema: "pa.Schema" + """The schema the defining query resolved to when the view was created.""" + namespace_path: List[str] = field(default_factory=list) + """The namespace holding the view; empty is the root namespace.""" + default_namespace_path: List[str] = field(default_factory=list) + """The namespace path those unqualified names resolve against. + + Recorded with the view because it outlives the session that declared it: a + reader resolving the query against its own default namespace could read a + different table than the view was defined over. + """ diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index cfe2405a4..c030ac1df 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -522,7 +522,7 @@ def test_stored_queries_and_legacy_layouts_are_read(): # A newer writer's layout is reported, never guessed at. for newer in ( - {"format": 2, "query": query}, + {"format": 3, "query": query}, {"kind": "select_v3", "source_table": "people"}, ): with pytest.raises(NotImplementedError, match="cannot refresh"): diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 1c887b218..c03862a49 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -2747,3 +2747,57 @@ def test_remote_job_handle_reports_its_own_detail(): "limit": 500, "filter": "state = 'claim_complete'", } + + +def test_view_crud_addresses_its_own_routes(): + # The view verbs are their own routes, and the schema comes back in the + # namespace spec's JSON encoding, decoded into a pyarrow schema. + paths = [] + + def handler(request): + paths.append((request.command, request.path)) + if request.path.endswith("/view/list"): + body = {"views": ["adults"]} + elif request.path.endswith("/drop"): + body = {} + else: + body = { + "name": "adults", + "namespace": ["analytics"], + "query": "SELECT name FROM people", + "default_database": "dev", + "default_namespace": ["analytics"], + "schema": { + "fields": [ + {"name": "name", "nullable": True, "type": {"type": "utf8"}} + ] + }, + } + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(body).encode()) + + with mock_lancedb_connection(handler) as db: + view = db.create_view( + "adults", "SELECT name FROM people", namespace_path=["analytics"] + ) + assert view.name == "adults" + assert view.namespace_path == ["analytics"] + assert view.query == "SELECT name FROM people" + assert view.default_database == "dev" + assert view.default_namespace_path == ["analytics"] + assert view.schema == pa.schema([pa.field("name", pa.utf8(), nullable=True)]) + + described = db.describe_view("adults", namespace_path=["analytics"]) + assert described.schema == view.schema + + assert db.list_views(namespace_path=["analytics"]) == ["adults"] + db.drop_view("adults", namespace_path=["analytics"]) + + assert paths == [ + ("POST", "/v1/view/analytics$adults/create"), + ("POST", "/v1/view/analytics$adults/describe"), + ("GET", "/v1/namespace/analytics/view/list"), + ("POST", "/v1/view/analytics$adults/drop"), + ] diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index a65e9cd45..909971ab2 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4,6 +4,7 @@ import ctypes import gc +import json import os import sys import threading @@ -17,6 +18,7 @@ from typing import Any, List from unittest.mock import patch import lancedb +from lancedb import table as table_module from lancedb.dependencies import _PANDAS_AVAILABLE from lancedb.index import BTree, FTS, HnswFlat, HnswPq, HnswSq, IvfPq import numpy as np @@ -3027,6 +3029,236 @@ async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection): assert rows == [{"id": "c", "j": '{"k":3}'}] +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_add_python_objects_to_json_column(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("payload", pa.json_()), + pa.field("metadata", pa.struct([("label", pa.string())])), + pa.field("tags", pa.list_(pa.string())), + ] + ) + table = mem_db.create_table("json_python_objects_add", schema=schema) + table.add( + [ + { + "id": 1, + "payload": {"foo": "bar", "count": 2}, + "metadata": {"label": "dict"}, + "tags": ["a", "b"], + }, + { + "id": 2, + "payload": { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + }, + "metadata": {"label": "nested"}, + "tags": ["c"], + }, + { + "id": 3, + "payload": ["x", "y", "z"], + "metadata": {"label": "list"}, + "tags": ["d", "e"], + }, + { + "id": 4, + "payload": '{"foo": "bar"}', + "metadata": {"label": "string"}, + "tags": ["f"], + }, + { + "id": 5, + "payload": None, + "metadata": {"label": "null"}, + "tags": ["g"], + }, + ] + ) + + rows = {row["id"]: row for row in table.to_arrow().to_pylist()} + assert json.loads(rows[1]["payload"]) == {"foo": "bar", "count": 2} + assert json.loads(rows[2]["payload"]) == { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + } + assert json.loads(rows[3]["payload"]) == ["x", "y", "z"] + assert json.loads(rows[4]["payload"]) == {"foo": "bar"} + assert rows[5]["payload"] is None + assert rows[1]["metadata"] == {"label": "dict"} + assert rows[1]["tags"] == ["a", "b"] + + matched = table.search().where("json_extract(payload, '$.count') = '2'").to_list() + assert [row["id"] for row in matched] == [1] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_merge_insert_python_objects_to_json_column(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("payload", pa.json_()), + pa.field("metadata", pa.struct([("label", pa.string())])), + pa.field("tags", pa.list_(pa.string())), + ] + ) + table = mem_db.create_table("json_python_objects_merge", schema=schema) + table.add( + [ + { + "id": 1, + "payload": {"old": True}, + "metadata": {"label": "old"}, + "tags": ["old"], + } + ] + ) + + table.merge_insert( + "id" + ).when_matched_update_all().when_not_matched_insert_all().execute( + [ + { + "id": 1, + "payload": {"foo": "bar", "count": 2}, + "metadata": {"label": "dict"}, + "tags": ["a", "b"], + }, + { + "id": 2, + "payload": { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + }, + "metadata": {"label": "nested"}, + "tags": ["c"], + }, + { + "id": 3, + "payload": ["x", "y", "z"], + "metadata": {"label": "list"}, + "tags": ["d", "e"], + }, + { + "id": 4, + "payload": '{"foo": "bar"}', + "metadata": {"label": "string"}, + "tags": ["f"], + }, + { + "id": 5, + "payload": None, + "metadata": {"label": "null"}, + "tags": ["g"], + }, + ] + ) + + rows = {row["id"]: row for row in table.to_arrow().to_pylist()} + assert json.loads(rows[1]["payload"]) == {"foo": "bar", "count": 2} + assert json.loads(rows[2]["payload"]) == { + "name": "alice", + "tags": ["x", "y"], + "nested": {"enabled": True}, + } + assert json.loads(rows[3]["payload"]) == ["x", "y", "z"] + assert json.loads(rows[4]["payload"]) == {"foo": "bar"} + assert rows[5]["payload"] is None + assert rows[1]["metadata"] == {"label": "dict"} + assert rows[1]["tags"] == ["a", "b"] + + matched = table.search().where("json_extract(payload, '$.count') = '2'").to_list() + assert [row["id"] for row in matched] == [1] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_add_python_objects_to_nested_json_fields(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field( + "info", + pa.struct([pa.field("payload", pa.json_())]), + ), + pa.field("documents", pa.list_(pa.field("item", pa.json_()))), + ] + ) + table = mem_db.create_table("nested_json_python_objects_add", schema=schema) + + table.add( + [ + { + "id": 1, + "info": {"payload": {"kind": "struct", "value": 1}}, + "documents": [{"kind": "list", "value": 2}], + } + ] + ) + + row = table.to_arrow().to_pylist()[0] + assert json.loads(row["info"]["payload"]) == {"kind": "struct", "value": 1} + assert json.loads(row["documents"][0]) == {"kind": "list", "value": 2} + assert [ + row["id"] + for row in table.search() + .where("json_extract(info.payload, '$.value') = '1'") + .to_list() + ] == [1] + assert [ + row["id"] + for row in table.search() + .where("json_extract(documents[1], '$.value') = '2'") + .to_list() + ] == [1] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_json_serialization_plan_skips_non_json_branches(): + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), 3)), + pa.field("payload", pa.json_()), + pa.field( + "metadata", + pa.struct( + [ + pa.field("label", pa.string()), + pa.field("payload", pa.json_()), + ] + ), + ), + pa.field( + "documents", + pa.list_( + pa.struct( + [ + pa.field("title", pa.string()), + pa.field("payload", pa.json_()), + ] + ) + ), + ), + ] + ) + + plans = table_module._json_serialization_plans(schema) + + assert set(plans) == {"payload", "metadata", "documents"} + metadata_children = plans["metadata"].children + assert metadata_children is not None + assert set(metadata_children) == {"payload"} + documents_item = plans["documents"].item + assert documents_item is not None + assert documents_item.children is not None + assert set(documents_item.children) == {"payload"} + + @pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") @pytest.mark.asyncio async def test_add_all_null_json_batch(mem_db_async: AsyncConnection): diff --git a/python/src/connection.rs b/python/src/connection.rs index a972f4351..69f54c683 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -13,7 +13,11 @@ use crate::{ runtime::future_into_py, table::Table, }; -use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow}; +use arrow::{ + datatypes::Schema, + ffi_stream::ArrowArrayStreamReader, + pyarrow::{FromPyArrow, ToPyArrow}, +}; use lancedb::{ connection::Connection as LanceConnection, connection::NamespaceClientPushdownOperation, @@ -100,6 +104,28 @@ fn parse_default_namespace_path(path: Option>) -> PyResult, String, String, Vec, Py); + +/// A view description as a plain tuple, with the schema converted to the +/// pyarrow schema the caller would get from any other lancedb API. The Python +/// layer names the fields; this keeps the binding free of a class that would +/// have to be kept in step with the Rust struct. +fn view_description_to_py(view: lancedb::view::ViewDescription) -> PyResult { + Python::attach(|py| { + let schema = view.schema.to_pyarrow(py)?.unbind(); + Ok(( + view.name, + view.namespace_path, + view.query, + view.default_database, + view.default_namespace_path, + schema, + )) + }) +} + #[pymethods] impl Connection { fn __repr__(&self) -> String { @@ -768,6 +794,21 @@ impl Connection { }) } + pub fn drop_function_async( + self_: PyRef<'_, Self>, + name: String, + version: String, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + inner + .drop_function_async(name, version) + .await + .infer_error() + .map(|(dropped, job)| (dropped, crate::job::Job::new(job))) + }) + } + #[pyo3(signature = (name, value, namespace_path=None))] pub fn create_secret( self_: PyRef<'_, Self>, @@ -847,6 +888,66 @@ impl Connection { }) } + #[pyo3(signature = (name, query, namespace_path=None))] + pub fn create_view( + self_: PyRef<'_, Self>, + name: String, + query: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + let view = inner + .create_view(name, query, &namespace_path) + .await + .infer_error()?; + view_description_to_py(view) + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn describe_view( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + let view = inner + .describe_view(name, &namespace_path) + .await + .infer_error()?; + view_description_to_py(view) + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn drop_view( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.drop_view(name, &namespace_path).await.infer_error() + }) + } + + #[pyo3(signature = (namespace_path=None))] + pub fn list_views( + self_: PyRef<'_, Self>, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.list_views(&namespace_path).await.infer_error() + }) + } + pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index a183b9e0d..64c675989 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.3" +version = "0.40.0-beta.6" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true @@ -106,6 +106,9 @@ sha2 = "0.10" [dev-dependencies] anyhow = "1" lance-testing = { workspace = true } +# RestAdapter is only used by cfg(test) code. Keep rest-adapter off the +# production remote feature so Python wheels do not resolve its Axum 0.7 stack. +lance-namespace-impls = { workspace = true, features = ["rest-adapter"] } tempfile = { workspace = true } random_word = { version = "0.4.3", features = ["en"] } tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] } @@ -169,7 +172,6 @@ remote = [ "dep:webbrowser", "dep:fs4", "lance-namespace-impls/rest", - "lance-namespace-impls/rest-adapter", ] # Publish LanceDB's internal metrics (currently object store request counts, # bytes, latency, errors, and throttles) through the `metrics` crate facade, diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 7234a911f..962e9e586 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -37,7 +37,11 @@ use crate::remote::{ }, }; use crate::secrets::SecretInfo; -use crate::utils::{validate_secret_component, validate_secret_reference}; +use crate::utils::{ + validate_namespace, validate_secret_component, validate_secret_reference, + validate_view_reference, +}; +use crate::view::ViewDescription; use lance::io::ObjectStoreParams; pub use lance_file::version::LanceFileVersion; #[cfg(feature = "remote")] @@ -649,6 +653,22 @@ impl Connection { .await } + /// Start dropping a Function and return its cleanup job. + /// + /// The name is unbound before this returns; the object's content may still be being + /// deleted. Await [`Job::wait`][crate::job::Job::wait] to wait for that to finish. When + /// the server deletes inline, or when nothing was bound, the returned job is already + /// finished and has no id. Local databases return [`Error::NotSupported`]. + pub async fn drop_function_async( + &self, + name: impl AsRef, + version: impl AsRef, + ) -> Result<(bool, crate::job::Job)> { + self.internal + .drop_function_async(name.as_ref(), version.as_ref()) + .await + } + /// Create a named Secret in this database. /// /// Fails if the name is taken, so a create can never silently become a @@ -731,6 +751,94 @@ impl Connection { .await } + /// Create a view: a named query the database plans on every read. + /// + /// The query is planned once, here, so one that cannot be planned is + /// refused now rather than at the first read. A view holds no rows, and + /// its readers see its sources as they are at read time. + /// + /// There is no replace: a name already taken is an error, and changing a + /// view is a drop followed by a create, each authorized against what it + /// actually touches. Local databases return [`Error::NotSupported`]. + /// + /// # Example + /// + /// ```no_run + /// # async fn view_lifecycle( + /// # connection: &lancedb::Connection, + /// # ) -> Result<(), Box> { + /// let namespace = vec!["analytics".to_string()]; + /// + /// let view = connection + /// .create_view( + /// "recent_orders", + /// "SELECT id, total FROM orders WHERE total > 100", + /// &namespace, + /// ) + /// .await?; + /// println!("{} has {} columns", view.name, view.schema.fields().len()); + /// + /// // The query comes back as it was recorded, with the defaults its + /// // unqualified names resolve against. + /// let described = connection.describe_view("recent_orders", &namespace).await?; + /// println!("{} in {:?}", described.query, described.default_namespace_path); + /// + /// let names = connection.list_views(&namespace).await?; + /// assert!(names.iter().any(|name| name == "recent_orders")); + /// + /// // Dropping the view leaves `orders` untouched. + /// connection.drop_view("recent_orders", &namespace).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn create_view( + &self, + name: impl AsRef, + query: impl AsRef, + namespace_path: &[String], + ) -> Result { + validate_view_reference(name.as_ref(), namespace_path)?; + self.internal + .create_view(name.as_ref(), query.as_ref(), namespace_path) + .await + } + + /// What this database records about one view: its defining query and the + /// schema that query resolved to. + /// + /// The schema is the one recorded at creation. A source altered since then + /// shows up when the view is read, not here. Local databases return + /// [`Error::NotSupported`]. + pub async fn describe_view( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result { + validate_view_reference(name.as_ref(), namespace_path)?; + self.internal + .describe_view(name.as_ref(), namespace_path) + .await + } + + /// Drop a view. + /// + /// The tables it reads are untouched: a view holds no rows of its own. + /// Local databases return [`Error::NotSupported`]. + pub async fn drop_view(&self, name: impl AsRef, namespace_path: &[String]) -> Result<()> { + validate_view_reference(name.as_ref(), namespace_path)?; + self.internal.drop_view(name.as_ref(), namespace_path).await + } + + /// The names of the views in one namespace. + /// + /// Names only; a definition is query metadata and comes from + /// [`Self::describe_view`]. The client walks all server pages before + /// returning. Local databases return [`Error::NotSupported`]. + pub async fn list_views(&self, namespace_path: &[String]) -> Result> { + validate_namespace(namespace_path)?; + self.internal.list_views(namespace_path).await + } + /// Rename a table in the database. /// /// This is only supported in LanceDB Cloud. diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 0adac778c..086d1df35 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -32,6 +32,7 @@ use crate::job::Job; use crate::materialized_view::CreateMaterializedViewRequest; use crate::secrets::SecretInfo; use crate::table::{BaseTable, WriteOptions}; +use crate::view::ViewDescription; pub mod listing; pub mod namespace; @@ -258,6 +259,12 @@ fn secret_catalog_not_supported() -> Result { }) } +fn view_ops_not_supported() -> Result { + Err(crate::error::Error::NotSupported { + message: "View operations are not supported by this database".to_string(), + }) +} + /// The `Database` trait defines the interface for database implementations. /// /// A database is responsible for managing tables and their metadata. @@ -393,6 +400,18 @@ pub trait Database: async fn drop_function(&self, _name: &str, _version: &str) -> Result { function_catalog_not_supported() } + /// Start dropping a Function and return a handle to the cleanup job. + /// + /// Backends without asynchronous cleanup complete the drop before returning an + /// already-finished job. + async fn drop_function_async( + &self, + name: &str, + version: &str, + ) -> Result<(bool, crate::job::Job)> { + let dropped = self.drop_function(name, version).await?; + Ok((dropped, crate::job::Job::new_done())) + } /// Create a named Secret in this database. Fails if the name is taken, so /// a create can never silently become a rotation. async fn create_secret( @@ -431,6 +450,35 @@ pub trait Database: async fn describe_secret(&self, _name: &str, _namespace_path: &[String]) -> Result { secret_catalog_not_supported() } + /// Create a view from a defining query. The query is planned once, by the + /// database, so one that cannot be planned is refused now rather than at + /// the first read. + async fn create_view( + &self, + _name: &str, + _query: &str, + _namespace_path: &[String], + ) -> Result { + view_ops_not_supported() + } + /// What the database records about one view: its defining query and the + /// schema that query resolved to. + async fn describe_view( + &self, + _name: &str, + _namespace_path: &[String], + ) -> Result { + view_ops_not_supported() + } + /// Drop a view. Its sources are untouched -- a view holds no rows of its + /// own. + async fn drop_view(&self, _name: &str, _namespace_path: &[String]) -> Result<()> { + view_ops_not_supported() + } + /// The names of the views in one namespace. + async fn list_views(&self, _namespace_path: &[String]) -> Result> { + view_ops_not_supported() + } /// Open a job by id, returning a handle with its record already /// populated. Fails with [`crate::Error::JobNotFound`] when the server has /// no such job. diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 4d9afc7c7..530328cbb 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -202,6 +202,7 @@ pub mod table; #[cfg(test)] pub mod test_utils; pub mod utils; +pub mod view; use std::{fmt::Display, str::FromStr}; diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 74d831c82..c4456dbe1 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -9,6 +9,8 @@ //! query this version cannot maintain reads back as unrefreshable, not as a //! plain table. Queries, indexes and search work on the view unchanged. +mod grouped; +pub use grouped::IVF_PARTITION; mod query; pub mod refresh; @@ -79,13 +81,21 @@ const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions"; /// produces, which is what lets a query embed its own text. const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions"; -/// The layout this version writes under [`DEFINITION_META_KEY`]: -/// `{"format": 1, "query": ""}`, the query as -/// [`MaterializedViewDefinition::to_sql`] renders it. A reader refuses a -/// newer format rather than guess at it. The layout also carries -/// `"kind": "query"`, which readers older than the format number report -/// as an unrefreshable view instead of failing to read the metadata. -pub const DEFINITION_FORMAT: u64 = 1; +/// The newest layout this version reads under [`DEFINITION_META_KEY`]: +/// `{"format": N, "query": ""}`, the query as +/// [`MaterializedViewDefinition::to_sql`] renders it. Format 2 is a query +/// with `GROUP BY`; every other query is written as format 1, so a reader +/// that predates grouping reports a grouped view as unrefreshable rather +/// than failing to parse it. A reader refuses a newer format rather than +/// guess at it. The layout also carries `"kind": "query"`, which readers +/// older than the format number report as an unrefreshable view instead of +/// failing to read the metadata. +pub const DEFINITION_FORMAT: u64 = 2; + +/// The format `definition` is written in; see [`DEFINITION_FORMAT`]. +fn format_of(definition: &MaterializedViewDefinition) -> u64 { + if definition.is_grouped() { 2 } else { 1 } +} /// Legacy `kind` tag of the structured layout written before /// [`DEFINITION_FORMAT`] existed; still read, never written. @@ -240,6 +250,9 @@ pub struct MaterializedViewDefinition { pub projections: Vec, /// SQL predicate selecting the rows the view holds. pub filter: Option, + /// `GROUP BY` expressions. A grouped view holds one row per group and is + /// recomputed in full whenever its source changes. + pub group_by: Vec, /// Cap on the number of rows the view holds, in materialization order. pub limit: Option, } @@ -250,7 +263,7 @@ impl MaterializedViewDefinition { /// ```sql /// SELECT , ... /// FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias] - /// [WHERE predicate] [LIMIT n] + /// [WHERE predicate] [GROUP BY expr, ...] [LIMIT n] /// ``` /// /// A Function in `FROM` position yields one row per element it returns; @@ -283,6 +296,11 @@ impl MaterializedViewDefinition { definition_to_metadata(self) } + /// Whether the query has a `GROUP BY`. + pub fn is_grouped(&self) -> bool { + !self.group_by.is_empty() + } + /// Whether the query is `SELECT *`. pub fn selects_star(&self) -> bool { matches!(self.projections.as_slice(), [p] if *p == ViewProjection::star()) @@ -359,7 +377,7 @@ struct LegacyDefinition { pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result { Ok(serde_json::json!({ "kind": "query", - "format": DEFINITION_FORMAT, + "format": format_of(definition), "query": definition.to_sql(), }) .to_string()) @@ -389,6 +407,13 @@ pub fn read_definition(metadata: &HashMap) -> Result) -> Result err, })?; + if definition.is_grouped() { + return grouped::plan(source_schema, definition, filter); + } // Projections are typed against the source, or for an unnested view // against the source with the list column replaced by its element // under the alias, where `c.chunk` is an ordinary nested path. @@ -616,34 +645,7 @@ pub(crate) fn plan( } if let Some(filter) = filter.as_deref() { - let expr = planner - .parse_filter(filter) - .map_err(|e| Error::InvalidInput { - message: format!("invalid view filter: {e}"), - })?; - ensure_immutable(&expr, |message| Error::InvalidInput { - message: format!("invalid view filter: {message}"), - })?; - let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput { - message: format!("invalid view filter: {message}"), - })?; - // A committed filter has to be usable as a predicate. - let read_schema = project_schema(&source_schema, &filter_inputs); - let data_type = Planner::new(read_schema.clone()) - .create_physical_expr(&expr) - .map_err(|e| Error::InvalidInput { - message: format!("invalid view filter: {e}"), - })? - .data_type(read_schema.as_ref()) - .map_err(|e| Error::InvalidInput { - message: format!("invalid view filter: {e}"), - })?; - if data_type != DataType::Boolean { - return Err(Error::InvalidInput { - message: format!("view filter must be a boolean predicate, not {data_type}"), - }); - } - inputs.extend(filter_inputs); + inputs.extend(plan_filter(&source_schema, filter)?); } let definition = MaterializedViewDefinition { @@ -655,6 +657,7 @@ pub(crate) fn plan( .map(|(output, expression)| ViewProjection { output, expression }) .collect(), filter, + group_by: Vec::new(), limit, }; let mut inputs: Vec = inputs @@ -671,6 +674,39 @@ pub(crate) fn plan( }) } +/// Check that `filter` is an immutable boolean predicate over `source_schema`, +/// returning the columns it reads. +pub(crate) fn plan_filter(source_schema: &SchemaRef, filter: &str) -> Result> { + let expr = Planner::new(source_schema.clone()) + .parse_filter(filter) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + ensure_immutable(&expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + let filter_inputs = resolve_inputs(source_schema, &expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + // A committed filter has to be usable as a predicate. + let read_schema = project_schema(source_schema, &filter_inputs); + let data_type = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })? + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + if data_type != DataType::Boolean { + return Err(Error::InvalidInput { + message: format!("view filter must be a boolean predicate, not {data_type}"), + }); + } + Ok(filter_inputs) +} + /// The schema a projection over an unnested view is planned against: the /// source's, with the list column replaced by its element type under the /// alias. @@ -1046,6 +1082,15 @@ impl PreparedDeclaration { /// read: the column the view projects it to, if any, otherwise an /// internal projection added here, named by [`input_column_name`]. pub fn input_column(&mut self, source_column: &str) -> Result { + // A grouped view's rows are groups; no source row carries a value into one. + if self.definition.is_grouped() { + return Err(Error::InvalidInput { + message: format!( + "a computed column on a grouped view cannot read source column \ + '{source_column}'; declare it on a view over the grouped view" + ), + }); + } if let Some(output) = self.lineage.get(source_column).and_then(|o| o.first()) { return Ok(output.clone()); } @@ -1357,6 +1402,7 @@ pub async fn prepare_declaration( .collect(), }, filter: filter.map(str::to_string), + group_by: Vec::new(), limit, }; prepare_definition(source, definition).await @@ -1383,6 +1429,26 @@ pub async fn prepare_declaration( /// # Ok(()) /// # } /// ``` +/// +/// A grouped view holds one row per group and is recomputed in full on +/// every refresh after a source change: +/// +/// ``` +/// # #![recursion_limit = "256"] +/// use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition}; +/// +/// # async fn declare(events: &lancedb::Table) -> Result<(), Box> { +/// let definition = MaterializedViewDefinition::from_sql( +/// "SELECT kind, count(*) AS n, array_agg(id) AS ids FROM events GROUP BY kind", +/// )?; +/// let view = prepare_definition(events, definition) +/// .await? +/// .create("events_by_kind") +/// .await?; +/// view.refresh().execute().await?; +/// # Ok(()) +/// # } +/// ``` pub async fn prepare_definition( source: &Table, definition: MaterializedViewDefinition, @@ -1532,6 +1598,9 @@ async fn prepare_with( lineage, .. } = plan(source_schema.clone(), &definition, staging.as_ref())?; + if definition.is_grouped() { + grouped::check(native.dataset.get().await?.as_ref(), &definition).await?; + } // What later projections (`input_column`) are planned against: for an // unnested view the flattened schema, where the alias is a column. let planning_schema = match physical_unnest(&definition, staging.as_ref())? { @@ -2067,6 +2136,7 @@ mod tests { }, ], filter: Some("age >= 18".into()), + group_by: Vec::new(), limit: Some(10), lateral: None, } @@ -3110,6 +3180,7 @@ mod tests { expression: "name".to_string(), }], filter: None, + group_by: Vec::new(), limit: None, } } @@ -3187,8 +3258,8 @@ mod tests { #[test] fn a_newer_format_is_reported_not_guessed() { assert_eq!( - read(r#"{"format":2,"query":"SELECT name FROM people"}"#).unwrap(), - Some(StoredDefinition::Newer { format: "2".into() }) + read(r#"{"format":3,"query":"SELECT name FROM people"}"#).unwrap(), + Some(StoredDefinition::Newer { format: "3".into() }) ); assert_eq!( read(r#"{"kind":"join"}"#).unwrap(), @@ -3201,6 +3272,7 @@ mod tests { r#"{"format":"one"}"#, r#"{"format":1}"#, r#"{"format":1,"query":"SELECT name FROM people GROUP BY name"}"#, + r#"{"format":2,"query":"SELECT name FROM people ORDER BY name"}"#, ] { assert!(read(stored).is_err(), "{stored}"); } @@ -3399,7 +3471,7 @@ mod tests { // The stored definition is the query alone; bindings live beside it. let stored: serde_json::Value = serde_json::from_str(&schema.metadata()[DEFINITION_META_KEY]).unwrap(); - assert_eq!(stored["format"], DEFINITION_FORMAT); + assert_eq!(stored["format"], 1); assert_eq!(view.definition().projections.len(), 2); assert_eq!(view.table().count_rows(None).await.unwrap(), 0); assert_eq!(conn.open_materialized_view("v").await.unwrap().name(), "v"); diff --git a/rust/lancedb/src/materialized_view/grouped.rs b/rust/lancedb/src/materialized_view/grouped.rs new file mode 100644 index 000000000..00e14527a --- /dev/null +++ b/rust/lancedb/src/materialized_view/grouped.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Views with `GROUP BY`: one row per group, planned and executed by +//! DataFusion, and recomputed in full whenever the source changes, since a +//! group spans fragments. + +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use arrow_array::cast::AsArray; +use arrow_array::{Array, FixedSizeListArray, UInt32Array}; +use arrow_buffer::NullBuffer; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef}; +use datafusion::catalog::default_table_source::provider_as_source; +use datafusion::datasource::empty::EmptyTable; +use datafusion::execution::FunctionRegistry; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::context::SessionState; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::prelude::SessionContext; +use datafusion_common::DataFusionError; +use datafusion_common::TableReference; +use datafusion_common::config::ConfigOptions; +use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; +use datafusion_expr::planner::{ContextProvider, ExprPlanner}; +use datafusion_expr::{ + AggregateUDF, ColumnarValue, Expr, HigherOrderUDF, LogicalPlan, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, TableSource, Volatility, WindowUDF, +}; +use datafusion_sql::planner::SqlToRel; +use datafusion_sql::sqlparser::dialect::GenericDialect; +use datafusion_sql::sqlparser::parser::Parser; +use futures::StreamExt; +use lance::Dataset; +use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use lance_core::ROW_ID; +use lance_datafusion::exec::SessionContextExt; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::vector::ivf::{IvfTransformer, new_ivf_transformer}; +use lance_linalg::distance::DistanceType; +use lance_linalg::kernels::normalize_fsl; +use uuid::Uuid; + +use super::refresh::to_view_batch; +use super::{ + MaterializedViewDefinition, Planned, SOURCE_ROW_ID_COLUMN, ViewProjection, ensure_immutable, + plan_filter, query, without_declarations, +}; +use crate::{Error, Result}; + +/// The name the source is planned under; never visible to the user. +const SOURCE: &str = "__source"; + +/// `ivf_partition(column)`: the partition of the IVF index on `column` that +/// each vector falls in, assigned by that index's centroids and distance +/// type. Bound per refresh, so a retrained index regroups the view. +pub const IVF_PARTITION: &str = "ivf_partition"; + +/// The index `ivf_partition(column)` assigns by. +#[derive(Debug, Clone)] +struct IvfBinding { + index: Uuid, + transformer: Arc, + /// A cosine index assigns L2 over unit vectors, as lance's index path does. + normalize: bool, +} + +/// `bindings` maps a source column to its index; planning runs unbound. +#[derive(Debug)] +struct IvfPartition { + signature: Signature, + bindings: HashMap, +} + +impl PartialEq for IvfPartition { + fn eq(&self, other: &Self) -> bool { + self.bindings.len() == other.bindings.len() + && self.bindings.iter().all(|(column, binding)| { + other + .bindings + .get(column) + .is_some_and(|b| b.index == binding.index) + }) + } +} + +impl Eq for IvfPartition {} + +impl std::hash::Hash for IvfPartition { + fn hash(&self, state: &mut H) { + let mut bound: Vec<_> = self.bindings.iter().map(|(c, b)| (c, b.index)).collect(); + bound.sort(); + bound.hash(state); + } +} + +impl ScalarUDFImpl for IvfPartition { + fn name(&self) -> &str { + IVF_PARTITION + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> datafusion_common::Result { + // Float vectors under L2/dot/cosine, byte vectors under Hamming; the + // binding checks the index's metric against the element type. + match arg_types { + [DataType::FixedSizeList(element, _)] + if element.data_type().is_floating() || *element.data_type() == DataType::UInt8 => + { + Ok(DataType::UInt32) + } + other => datafusion_common::plan_err!( + "{IVF_PARTITION} takes one vector column, not {other:?}" + ), + } + } + + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> datafusion_common::Result { + // The argument is a bare column (checked at planning), so its field + // names the column the binding was made for. + let column = args.arg_fields[0].name(); + let Some(binding) = self.bindings.get(column) else { + return datafusion_common::exec_err!("{IVF_PARTITION}({column}) has no index bound"); + }; + let vectors = args.args[0].to_array(args.number_rows)?; + let vectors = vectors.as_fixed_size_list(); + let normalized; + let vectors = if binding.normalize { + normalized = + normalize_fsl(vectors).map_err(|e| DataFusionError::External(Box::new(e)))?; + &normalized + } else { + vectors + }; + let partitions = binding + .transformer + .compute_partitions(vectors) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + // A null vector falls in no partition, and neither does one the + // assigner could not place (a zero vector under cosine): a NULL bucket + // is honest, membership in bucket 0 is not. + let partitions = UInt32Array::new( + partitions.values().clone(), + NullBuffer::union(vectors.nulls(), partitions.nulls()), + ); + Ok(ColumnarValue::Array(Arc::new(partitions))) + } +} + +fn session(bindings: HashMap) -> SessionState { + let mut state = SessionStateBuilder::new().with_default_features().build(); + let ivf = IvfPartition { + signature: Signature::any(1, Volatility::Immutable), + bindings, + }; + state + .register_udf(Arc::new(ScalarUDF::new_from_impl(ivf))) + .expect("registering a scalar function cannot fail"); + state +} + +/// The columns `plan` calls `ivf_partition` on. The argument must be a bare +/// column: its field is how the bound function finds the column's index. +fn ivf_columns(plan: &LogicalPlan) -> Result> { + let mut columns = Vec::new(); + let mut misuse = None; + plan.apply(|node| { + for expr in node.expressions() { + expr.apply(|e| { + if let Expr::ScalarFunction(call) = e + && call.name() == IVF_PARTITION + { + match call.args.as_slice() { + [Expr::Column(column)] => columns.push(column.name.clone()), + _ => misuse = Some(e.to_string()), + } + } + Ok(TreeNodeRecursion::Continue) + })?; + } + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| Error::InvalidInput { + message: format!("invalid grouped view: {e}"), + })?; + if let Some(call) = misuse { + return Err(Error::InvalidInput { + message: format!("{IVF_PARTITION} takes a vector column of the source, not `{call}`"), + }); + } + columns.sort(); + columns.dedup(); + Ok(columns) +} + +/// Refuse a grouped `definition` whose functions cannot be bound over +/// `source`, such as `ivf_partition` on a column without an IVF index. +pub(super) async fn check(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> { + bind(source, definition).await.map(|_| ()) +} + +/// Bind every `ivf_partition(column)` in `definition` to the IVF index on +/// that column of `source`. +async fn bind( + source: &Dataset, + definition: &MaterializedViewDefinition, +) -> Result> { + let schema = Arc::new(ArrowSchema::from(source.schema())); + let planned = logical_plan(&session(HashMap::new()), empty_source(&schema), definition)?; + let mut bindings = HashMap::new(); + for column in ivf_columns(&planned)? { + bindings.insert(column.clone(), bind_column(source, &column).await?); + } + Ok(bindings) +} + +async fn bind_column(source: &Dataset, column: &str) -> Result { + let field = source + .schema() + .field(column) + .ok_or_else(|| Error::InvalidInput { + message: format!("{IVF_PARTITION}: the source has no column '{column}'"), + })?; + // One index, whose segments all assign by one model. A segment trained + // on its own fragments carries its own centroids, and grouping every row + // by one segment's model would bucket the other segments' rows wrongly. + let mut found: Option<(String, Uuid, DistanceType, FixedSizeListArray)> = None; + for index in source.load_indices().await?.iter() { + if index.fields != [field.id] { + continue; + } + let Ok(vector) = source + .open_vector_index(column, &index.uuid, &NoOpMetricsCollector) + .await + else { + continue; + }; + let Some(centroids) = vector.ivf_model().centroids.clone() else { + continue; + }; + let metric = vector.metric_type(); + match &found { + None => found = Some((index.name.clone(), index.uuid, metric, centroids)), + Some((name, _, seen_metric, seen)) if *name == index.name => { + if *seen_metric != metric || seen.to_data() != centroids.to_data() { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}): the segments of index '{name}' were \ + trained separately and assign by different models; rebuild \ + the index before grouping by it" + ), + }); + } + } + Some((name, ..)) => { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}) is ambiguous: indices '{name}' and '{}' \ + both cover it", + index.name + ), + }); + } + } + } + let byte_vectors = matches!( + field.data_type(), + DataType::FixedSizeList(element, _) if *element.data_type() == DataType::UInt8 + ); + let Some((_, uuid, metric, centroids)) = found else { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}) needs an IVF vector index on '{column}'; create one first" + ), + }); + }; + // The assigner pairs byte vectors with Hamming and float vectors with the + // rest; an index of the other kind cannot place this column's values. + if byte_vectors != (metric == DistanceType::Hamming) { + return Err(Error::InvalidInput { + message: format!( + "{IVF_PARTITION}({column}): the index assigns by {metric}, which does not \ + apply to {}", + field.data_type() + ), + }); + } + // Lance's index path assigns cosine by L2 over unit vectors. + let normalize = metric == DistanceType::Cosine; + let distance = if normalize { DistanceType::L2 } else { metric }; + Ok(IvfBinding { + index: uuid, + transformer: Arc::new(new_ivf_transformer(centroids, distance, vec![])), + normalize, + }) +} + +fn empty_source(source_schema: &SchemaRef) -> Arc { + let mut fields = source_schema.fields().to_vec(); + fields.push(Arc::new(ArrowField::new(ROW_ID, DataType::UInt64, false))); + provider_as_source(Arc::new(EmptyTable::new(Arc::new(ArrowSchema::new( + fields, + ))))) +} + +/// The query DataFusion runs: the view's projections plus the group's +/// smallest source row id, which stands as the row's provenance. The filter +/// is not here; the lance scan applies it, as for any other view. +fn sql(definition: &MaterializedViewDefinition) -> String { + let items: Vec = definition + .projections + .iter() + .map(|p| format!("{} AS {}", p.expression, query::ident_sql(&p.output))) + .collect(); + format!( + "SELECT {}, min({ROW_ID}) AS {ROW_ID} FROM {SOURCE} GROUP BY {}", + items.join(", "), + definition.group_by.join(", ") + ) +} + +struct Provider<'a> { + state: &'a SessionState, + source: Arc, +} + +impl ContextProvider for Provider<'_> { + fn get_table_source( + &self, + name: TableReference, + ) -> datafusion_common::Result> { + if name.schema().is_none() && name.table() == SOURCE { + Ok(self.source.clone()) + } else { + datafusion_common::plan_err!("a grouped view reads only its source, not '{name}'") + } + } + fn get_expr_planners(&self) -> &[Arc] { + self.state.expr_planners() + } + fn get_function_meta(&self, name: &str) -> Option> { + self.state.scalar_functions().get(name).cloned() + } + fn get_higher_order_meta(&self, name: &str) -> Option> { + self.state.higher_order_functions().get(name).cloned() + } + fn get_aggregate_meta(&self, name: &str) -> Option> { + self.state.aggregate_functions().get(name).cloned() + } + fn get_window_meta(&self, name: &str) -> Option> { + self.state.window_functions().get(name).cloned() + } + fn get_variable_type(&self, _: &[String]) -> Option { + None + } + fn options(&self) -> &ConfigOptions { + self.state.config_options() + } + fn udf_names(&self) -> Vec { + self.state.scalar_functions().keys().cloned().collect() + } + fn higher_order_function_names(&self) -> Vec { + self.state + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn udaf_names(&self) -> Vec { + self.state.aggregate_functions().keys().cloned().collect() + } + fn udwf_names(&self) -> Vec { + self.state.window_functions().keys().cloned().collect() + } +} + +fn logical_plan( + state: &SessionState, + source: Arc, + definition: &MaterializedViewDefinition, +) -> Result { + let invalid = |e: &dyn std::fmt::Display| Error::InvalidInput { + message: format!("invalid grouped view: {e}"), + }; + let statement = Parser::parse_sql(&GenericDialect {}, &sql(definition)) + .map_err(|e| invalid(&e))? + .pop() + .ok_or_else(|| invalid(&"empty query"))?; + SqlToRel::new(&Provider { state, source }) + .sql_statement_to_plan(statement) + .map_err(|e| invalid(&e)) +} + +/// Plan a grouped definition against the source schema. `filter` is the +/// canonical filter, already spelled the way [`super::plan`] stores it. +pub(super) fn plan( + source_schema: SchemaRef, + definition: &MaterializedViewDefinition, + filter: Option, +) -> Result { + query::check_grouping(definition)?; + let mut inputs = match &filter { + Some(filter) => plan_filter(&source_schema, filter)?, + None => Vec::new(), + }; + let mut projections: Vec = Vec::with_capacity(definition.projections.len()); + for p in &definition.projections { + if p.output == SOURCE_ROW_ID_COLUMN || p.output == ROW_ID { + return Err(Error::InvalidInput { + message: format!("view column name '{}' is reserved", p.output), + }); + } + if projections.iter().any(|seen| seen.output == p.output) { + return Err(Error::ColumnAlreadyExists { + name: p.output.clone(), + }); + } + let expression = + query::canonical_expr(&p.expression).map_err(|e| Error::InvalidExpression { + column: p.output.clone(), + message: e.to_string(), + })?; + projections.push(ViewProjection { + output: p.output.clone(), + expression, + }); + } + let group_by = definition + .group_by + .iter() + .map(|key| query::canonical_expr(key)) + .collect::>>()?; + let definition = MaterializedViewDefinition { + projections, + filter, + group_by, + ..definition.clone() + }; + + let planned = logical_plan( + &session(HashMap::new()), + empty_source(&source_schema), + &definition, + )?; + ivf_columns(&planned)?; + + let mut exprs = Vec::new(); + planned + .apply(|node| { + exprs.extend(node.expressions()); + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| Error::InvalidInput { + message: format!("invalid grouped view: {e}"), + })?; + for expr in &exprs { + ensure_immutable(expr, |message| Error::InvalidInput { + message: format!("invalid grouped view: {message}"), + })?; + inputs.extend( + expr.column_refs() + .into_iter() + .map(|c| c.name.clone()) + .filter(|name| name != ROW_ID && source_schema.field_with_name(name).is_ok()), + ); + } + inputs.sort(); + inputs.dedup(); + + let fields = planned + .schema() + .fields() + .iter() + .filter(|f| f.name() != ROW_ID) + .map(|f| without_declarations(f)) + .collect(); + Ok(Planned { + definition, + fields, + lineage: HashMap::new(), + inputs, + }) +} + +/// Every group of `source`, as view rows in `schema`. +pub(super) async fn stream( + source: &Dataset, + definition: &MaterializedViewDefinition, + inputs: &[String], + schema: SchemaRef, + rows_written: Arc, +) -> Result { + let mut scanner = source.scan(); + scanner.with_row_id(); + if let Some(filter) = &definition.filter { + scanner.filter(filter)?; + } + scanner.project(inputs)?; + let scan: SendableRecordBatchStream = scanner.try_into_stream().await?.into(); + + let state = session(bind(source, definition).await?); + let ctx = SessionContext::new_with_state(state.clone()); + let source = ctx.read_one_shot(scan)?.into_view(); + let planned = logical_plan(&state, provider_as_source(source), definition)?; + let groups = ctx + .execute_logical_plan(planned) + .await? + .execute_stream() + .await?; + + let out_schema = schema.clone(); + let mapped = groups.map(move |batch| { + let batch = to_view_batch(&batch?, &out_schema)?; + rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(batch) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) +} diff --git a/rust/lancedb/src/materialized_view/query.rs b/rust/lancedb/src/materialized_view/query.rs index 35af77f0b..c1a987b2c 100644 --- a/rust/lancedb/src/materialized_view/query.rs +++ b/rust/lancedb/src/materialized_view/query.rs @@ -18,8 +18,9 @@ //! query: fail closed, never materialize it at the wrong cardinality. use datafusion_sql::sqlparser::ast::{ - Expr, FunctionArg, FunctionArgExpr, JoinOperator, LimitClause, ObjectName, ObjectNamePart, - Query, SelectItem, SetExpr, Statement, TableFactor, TableFunctionArgs, TableWithJoins, Value, + Expr, FunctionArg, FunctionArgExpr, GroupByExpr, JoinOperator, LimitClause, ObjectName, + ObjectNamePart, Query, SelectItem, SetExpr, Statement, TableFactor, TableFunctionArgs, + TableWithJoins, Value, }; use datafusion_sql::sqlparser::dialect::GenericDialect; use datafusion_sql::sqlparser::keywords::{ @@ -40,7 +41,8 @@ fn invalid(message: impl Into) -> Error { } const SHAPE: &str = "a materialized view is defined by `SELECT columns FROM table \ - [, function(args) AS alias | , UNNEST(column) AS alias] [WHERE predicate] [LIMIT n]`"; + [, function(args) AS alias | , UNNEST(column) AS alias] [WHERE predicate] \ + [GROUP BY expr, ...] [LIMIT n]`"; /// Whether `name` must be delimited to read back as this identifier: bare, /// the parser would take it as a keyword, or a different spelling. @@ -394,14 +396,59 @@ fn extract(query: &Query) -> Result { Some(_) => return Err(invalid("view limit must be a plain `LIMIT n`")), }; - Ok(MaterializedViewDefinition { + if select.having.is_some() { + return Err(invalid("HAVING is not supported in a view query")); + } + let group_by = match &select.group_by { + GroupByExpr::Expressions(exprs, modifiers) + if modifiers.is_empty() + && !exprs.iter().any(|e| { + matches!(e, Expr::Rollup(_) | Expr::Cube(_) | Expr::GroupingSets(_)) + }) => + { + exprs.iter().map(|e| e.to_string()).collect() + } + _ => { + return Err(invalid( + "a view groups by a plain `GROUP BY expr, ...` list", + )); + } + }; + + let definition = MaterializedViewDefinition { source_table, source_namespace, lateral, projections, filter: select.selection.as_ref().map(|e| e.to_string()), + group_by, limit, - }) + }; + check_grouping(&definition)?; + Ok(definition) +} + +/// The clauses a grouped view cannot combine with: each would have to be +/// maintained per group rather than per source row. +pub fn check_grouping(definition: &MaterializedViewDefinition) -> Result<()> { + if !definition.is_grouped() { + return Ok(()); + } + if definition.lateral.is_some() { + return Err(invalid( + "GROUP BY cannot be combined with a FROM-position item; group in one view \ + and expand in a view over it", + )); + } + if definition.selects_star() { + return Err(invalid( + "a grouped view selects its keys and aggregates, not `*`", + )); + } + if definition.limit.is_some() { + return Err(invalid("LIMIT is not supported together with GROUP BY")); + } + Ok(()) } /// The canonical query for `definition`; [`parse`] reads it back equal. @@ -451,6 +498,9 @@ pub fn render(definition: &MaterializedViewDefinition) -> String { if let Some(filter) = &definition.filter { sql.push_str(&format!(" WHERE {filter}")); } + if definition.is_grouped() { + sql.push_str(&format!(" GROUP BY {}", definition.group_by.join(", "))); + } if let Some(limit) = definition.limit { sql.push_str(&format!(" LIMIT {limit}")); } @@ -489,6 +539,10 @@ mod tests { "SELECT id, c.text FROM docs, chunk(upper(body)) AS c", ), ("SELECT * FROM `select`.t", "SELECT * FROM `select`.t"), + ( + "select k, array_agg(id) as ids, count(*) n from t where x > 1 group by k", + "SELECT k, array_agg(id) AS ids, count(*) AS n FROM t WHERE x > 1 GROUP BY k", + ), ( "SELECT meta.title AS title, id AS id FROM t", "SELECT meta.title, id FROM t", @@ -547,7 +601,6 @@ mod tests { #[test] fn unsupported_clauses_are_refused() { for sql in [ - "SELECT id FROM t GROUP BY id", "SELECT id FROM t ORDER BY id", "SELECT DISTINCT id FROM t", "SELECT id FROM t LIMIT 5 OFFSET 2", @@ -567,6 +620,12 @@ mod tests { "SELECT *, id FROM t", "WITH q AS (SELECT 1) SELECT id FROM t", "SELECT id FROM t HAVING id > 1", + "SELECT k, count(*) AS n FROM t GROUP BY k HAVING count(*) > 1", + "SELECT k FROM t GROUP BY ALL", + "SELECT k FROM t GROUP BY ROLLUP (k)", + "SELECT * FROM t GROUP BY k", + "SELECT k FROM t GROUP BY k LIMIT 5", + "SELECT k, c.x FROM t, UNNEST(xs) AS c GROUP BY k", ] { assert!(parse(sql).is_err(), "{sql}"); } diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 0fc9b99c5..6f19c3c91 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -297,6 +297,22 @@ pub(crate) async fn execute_refresh( }); } + // A group spans fragments: there is no per-fragment increment. + if definition.is_grouped() { + return rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + &inputs, + None, + persist.is_some(), + expected_incarnation, + ) + .await; + } let watermark = watermark.filter(|_| view_intact); match plan_increment( &source_ds, @@ -629,9 +645,17 @@ fn same_meaning( || stored.limit != replanned.limit || stored.projections.len() != replanned.projections.len() || stored.filter.is_some() != replanned.filter.is_some() + || stored.group_by.len() != replanned.group_by.len() { return false; } + // Lance's planner does not read aggregates; a grouped definition is + // compared in its canonical spelling, which planning produced. + if stored.is_grouped() { + return stored.projections == replanned.projections + && stored.group_by == replanned.group_by + && stored.filter == replanned.filter; + } let schema = match unnest { None => source_schema.clone(), Some(unnest) => match super::flattened_schema(source_schema, unnest) { @@ -1288,6 +1312,9 @@ async fn compute_stream( schema: SchemaRef, rows_written: Arc, ) -> Result { + if definition.is_grouped() { + return super::grouped::stream(source, definition, inputs, schema, rows_written).await; + } let RowScope { fragments, created_after, @@ -1363,31 +1390,42 @@ async fn compute_stream( None => batch, Some(expanded) => expanded.apply(&batch)?, }; - let mut columns = Vec::with_capacity(out_schema.fields().len()); - for field in out_schema.fields() { - if computed_column_from_field(field).is_some() { - columns.push(new_null_array(field.data_type(), batch.num_rows())); - continue; - } - let name = if field.name() == SOURCE_ROW_ID_COLUMN { - ROW_ID - } else { - field.name() - }; - let column = batch.column_by_name(name).ok_or_else(|| { - DataFusionError::Internal(format!( - "view column '{}' is not produced by the view's definition", - field.name() - )) - })?; - columns.push(column.clone()); - } + let batch = to_view_batch(&batch, &out_schema)?; rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); - Ok(RecordBatch::try_new(out_schema.clone(), columns)?) + Ok(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) } +/// `batch` in the view's physical schema: the row id becomes the +/// provenance column and computed columns are written NULL for their owner +/// to fill. +pub(super) fn to_view_batch( + batch: &RecordBatch, + out_schema: &SchemaRef, +) -> datafusion::common::Result { + let mut columns = Vec::with_capacity(out_schema.fields().len()); + for field in out_schema.fields() { + if computed_column_from_field(field).is_some() { + columns.push(new_null_array(field.data_type(), batch.num_rows())); + continue; + } + let name = if field.name() == SOURCE_ROW_ID_COLUMN { + ROW_ID + } else { + field.name() + }; + let column = batch.column_by_name(name).ok_or_else(|| { + DataFusionError::Internal(format!( + "view column '{}' is not produced by the view's definition", + field.name() + )) + })?; + columns.push(column.clone()); + } + Ok(RecordBatch::try_new(out_schema.clone(), columns)?) +} + /// The post-scan half of an unnested view's refresh: the scan reads /// `raw_inputs` plus the row id, and each batch is unnested on the list /// column, filtered, then projected by expressions typed against @@ -3641,6 +3679,7 @@ mod tests { }, ], filter: None, + group_by: Vec::new(), limit: None, lateral: None, }; @@ -3675,6 +3714,7 @@ mod tests { expression: "x".into(), }], filter: None, + group_by: Vec::new(), limit: None, lateral: None, }; @@ -4304,4 +4344,447 @@ mod tests { 3 ); } + + async fn grouped_source(conn: &Connection) -> Table { + conn.create_table( + "src", + record_batch!(("k", Int32, [1, 1, 2, 3]), ("x", Int32, [10, 11, 20, 30])).unwrap(), + ) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap() + } + + async fn declare(source: &Table, name: &str, sql: &str) -> Result { + crate::materialized_view::prepare_definition( + source, + MaterializedViewDefinition::from_sql(sql)?, + ) + .await? + .create(name) + .await + } + + /// Rows of `table` as `columns` rendered to text, sorted. + async fn rows(table: &Table, columns: &[&str]) -> Vec { + let batches = table + .query() + .select(Select::columns(columns)) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut rows = Vec::new(); + for batch in &batches { + let formatters: Vec<_> = batch + .columns() + .iter() + .map(|c| { + arrow_cast::display::ArrayFormatter::try_new(c.as_ref(), &Default::default()) + .unwrap() + }) + .collect(); + for row in 0..batch.num_rows() { + let cells: Vec = formatters + .iter() + .map(|f| f.value(row).to_string()) + .collect(); + rows.push(cells.join(" ")); + } + } + rows.sort(); + rows + } + + #[tokio::test] + async fn a_grouped_view_holds_one_row_per_group() { + let conn = connect("memory://").execute().await.unwrap(); + let source = grouped_source(&conn).await; + let view = declare( + &source, + "groups", + "SELECT k, array_agg(x) AS xs, sum(x) AS total FROM src WHERE x < 30 GROUP BY k", + ) + .await + .unwrap(); + assert_eq!( + view.definition().to_sql(), + "SELECT k, array_agg(x) AS xs, sum(x) AS total FROM src WHERE x < 30 GROUP BY k" + ); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 2); + assert_eq!(rows(view.table(), &["k", "total"]).await, ["1 21", "2 20"]); + let unchanged = view.refresh().execute().await.unwrap(); + assert_eq!(unchanged.mode, RefreshMode::NoOp); + + // Any source change recomputes every group. + source + .add(record_batch!(("k", Int32, [2]), ("x", Int32, [5])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(rows(view.table(), &["k", "total"]).await, ["1 21", "2 25"]); + + // A view over the groups expands them again. + let expanded = declare( + view.table(), + "elements", + "SELECT k, e AS x FROM groups, UNNEST(xs) AS e", + ) + .await + .unwrap(); + expanded.refresh().execute().await.unwrap(); + assert_eq!( + rows(expanded.table(), &["k", "x"]).await, + ["1 10", "1 11", "2 20", "2 5"] + ); + } + + /// Provenance is one source row per group, so the view's row-id keys + /// stay unique. + #[tokio::test] + async fn a_grouped_view_records_one_source_row_per_group() { + let conn = connect("memory://").execute().await.unwrap(); + let source = grouped_source(&conn).await; + let view = declare( + &source, + "groups", + "SELECT k, count(*) AS n FROM src GROUP BY k", + ) + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + let ids = rows(view.table(), &[SOURCE_ROW_ID_COLUMN]).await; + assert_eq!(ids.len(), 3); + assert_eq!(ids.iter().collect::>().len(), 3, "{ids:?}"); + } + + #[tokio::test] + async fn a_grouped_view_is_planned_at_declaration() { + let conn = connect("memory://").execute().await.unwrap(); + let source = grouped_source(&conn).await; + for sql in [ + // x is neither a key nor aggregated + "SELECT k, x FROM src GROUP BY k", + "SELECT k, random() AS r FROM src GROUP BY k", + "SELECT k, count(*) AS n FROM src GROUP BY missing", + "SELECT k, count(*) AS __source_row_id FROM src GROUP BY k", + "SELECT k, count(*) AS k FROM src GROUP BY k", + ] { + assert!(declare(&source, "v", sql).await.is_err(), "{sql}"); + } + let mut grouped = crate::materialized_view::prepare_definition( + &source, + MaterializedViewDefinition::from_sql("SELECT k, count(*) AS n FROM src GROUP BY k") + .unwrap(), + ) + .await + .unwrap(); + assert!(grouped.input_column("x").is_err()); + } + + /// Twenty nonzero vectors in two clusters far apart, ids 0-9 and 10-19. + /// Nonzero so every vector has a cosine direction. + async fn clustered_source(conn: &Connection) -> Table { + use arrow_array::types::Float32Type; + use arrow_array::{FixedSizeListArray, Int32Array}; + + let vectors = FixedSizeListArray::from_iter_primitive::( + (0..20).map(|i| { + let base = if i < 10 { 0.0 } else { 100.0 }; + Some(vec![Some(base + 1.0 + i as f32 * 0.1), Some(base)]) + }), + 2, + ); + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(0..20)) as arrow_array::ArrayRef, + ), + ("vec", Arc::new(vectors) as arrow_array::ArrayRef), + ]) + .unwrap(); + conn.create_table("images", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap() + } + + #[tokio::test] + async fn ivf_partition_groups_by_the_index_on_the_column() { + use crate::index::vector::IvfFlatIndexBuilder; + + const BUCKETS: &str = "SELECT ivf_partition(vec) AS bucket, count(*) AS n, \ + min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(vec)"; + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + let unindexed = declare(&source, "buckets", BUCKETS).await.unwrap_err(); + assert!( + unindexed.to_string().contains("IVF vector index"), + "{unindexed}" + ); + + source + .create_index( + &["vec"], + Index::IvfFlat(IvfFlatIndexBuilder::default().num_partitions(2)), + ) + .execute() + .await + .unwrap(); + let view = declare(&source, "buckets", BUCKETS).await.unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!( + rows(view.table(), &["n", "lo", "hi"]).await, + ["10 0 9", "10 10 19"] + ); + let buckets = rows(view.table(), &["bucket"]).await; + assert_eq!(buckets, ["0", "1"]); + } + + #[tokio::test] + async fn ivf_partition_takes_a_column() { + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + for sql in [ + "SELECT ivf_partition(id) AS b, count(*) AS n FROM images GROUP BY ivf_partition(id)", + "SELECT ivf_partition(vec, vec) AS b, count(*) AS n FROM images \ + GROUP BY ivf_partition(vec, vec)", + ] { + assert!(declare(&source, "v", sql).await.is_err(), "{sql}"); + } + } + + /// A cosine index assigns by L2 over unit vectors, as lance's own index + /// path does; forwarding cosine to the assigner panics. + #[tokio::test] + async fn ivf_partition_supports_cosine_indices() { + use crate::index::vector::IvfFlatIndexBuilder; + + const BUCKETS: &str = "SELECT ivf_partition(vec) AS bucket, count(*) AS n, \ + min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(vec)"; + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + source + .create_index( + &["vec"], + Index::IvfFlat( + IvfFlatIndexBuilder::default() + .num_partitions(2) + .distance_type(crate::DistanceType::Cosine), + ), + ) + .execute() + .await + .unwrap(); + let view = declare(&source, "cosine_buckets", BUCKETS).await.unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!( + rows(view.table(), &["n", "lo", "hi"]).await, + ["10 0 9", "10 10 19"] + ); + } + + /// Segments of one index trained separately carry different centroids; + /// grouping by one of them would bucket the other segments' rows wrongly. + #[tokio::test] + async fn ivf_partition_refuses_segments_with_different_models() { + use arrow_array::types::Float32Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array}; + use lance::index::DatasetIndexExt; + use lance::index::vector::VectorIndexParams; + use lance_index::IndexType; + + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + // A second fragment far from the first, so each segment trains its own centroids. + let far = FixedSizeListArray::from_iter_primitive::( + (0..20).map(|i| Some(vec![Some(-500.0 - i as f32), Some(-500.0)])), + 2, + ); + source + .add( + RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(20..40)) as ArrayRef, + ), + ("vec", Arc::new(far) as ArrayRef), + ]) + .unwrap(), + ) + .execute() + .await + .unwrap(); + let dataset = source.as_native().unwrap().dataset.get().await.unwrap(); + let mut dataset = dataset.as_ref().clone(); + let params = VectorIndexParams::ivf_flat(2, lance_linalg::distance::DistanceType::L2); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + let segment = dataset + .create_index_builder(&["vec"], IndexType::Vector, ¶ms) + .name("shared".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + segments.push(segment); + } + dataset + .commit_existing_index_segments("shared", "vec", segments) + .await + .unwrap(); + assert_eq!(dataset.load_indices().await.unwrap().len(), 2); + + let source = conn.open_table("images").execute().await.unwrap(); + let error = declare( + &source, + "buckets", + "SELECT ivf_partition(vec) AS b, count(*) AS n FROM images GROUP BY ivf_partition(vec)", + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("trained separately"), "{error}"); + } + + /// A vector the assigner cannot place has no bucket: it is grouped under + /// NULL, never under partition 0. + #[tokio::test] + async fn ivf_partition_leaves_an_unassignable_vector_unbucketed() { + use crate::index::vector::IvfFlatIndexBuilder; + use arrow_array::types::Float32Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array}; + + let conn = connect("memory://").execute().await.unwrap(); + let source = clustered_source(&conn).await; + source + .create_index( + &["vec"], + Index::IvfFlat( + IvfFlatIndexBuilder::default() + .num_partitions(2) + .distance_type(crate::DistanceType::Cosine), + ), + ) + .execute() + .await + .unwrap(); + // The zero vector has no direction, so cosine cannot place it. + let zero = FixedSizeListArray::from_iter_primitive::( + [Some(vec![Some(0.0), Some(0.0)])], + 2, + ); + source + .add( + RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![20])) as ArrayRef), + ("vec", Arc::new(zero) as ArrayRef), + ]) + .unwrap(), + ) + .execute() + .await + .unwrap(); + let view = declare( + &source, + "buckets", + "SELECT ivf_partition(vec) AS bucket, count(*) AS n, min(id) AS lo, max(id) AS hi \ + FROM images GROUP BY ivf_partition(vec)", + ) + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + // ArrayFormatter renders NULL as the empty string; partition numbers + // are the index's own and not asserted. + let groups = rows(view.table(), &["bucket", "n", "lo", "hi"]).await; + let (unbucketed, bucketed): (Vec<_>, Vec<_>) = + groups.iter().partition(|row| row.starts_with(' ')); + assert_eq!(unbucketed, [" 1 20 20"], "{groups:?}"); + let mut clusters: Vec<&str> = bucketed + .iter() + .map(|row| row.split_once(' ').unwrap().1) + .collect(); + clusters.sort(); + assert_eq!(clusters, ["10 0 9", "10 10 19"], "{groups:?}"); + } + + /// Byte vectors group by a Hamming index, the phash case. The centroids + /// are given, as lance's own tests do: two-means over a handful of + /// hashes can converge with one partition empty. + #[tokio::test] + async fn ivf_partition_groups_byte_vectors_by_hamming() { + use arrow_array::types::UInt8Type; + use arrow_array::{ArrayRef, FixedSizeListArray, Int32Array}; + use lance::index::DatasetIndexExt; + use lance::index::vector::VectorIndexParams; + use lance_index::IndexType; + use lance_index::vector::ivf::IvfBuildParams; + + let hashes = FixedSizeListArray::from_iter_primitive::( + (0..20u8).map(|i| { + let base = if i < 10 { 0x00 } else { 0xff }; + Some(vec![ + Some(base ^ (i % 4)), + Some(base), + Some(base), + Some(base), + ]) + }), + 4, + ); + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(0..20)) as ArrayRef, + ), + ("phash", Arc::new(hashes) as ArrayRef), + ]) + .unwrap(); + let conn = connect("memory://").execute().await.unwrap(); + let source = conn + .create_table("images", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let centroids = FixedSizeListArray::from_iter_primitive::( + [Some(vec![Some(0); 4]), Some(vec![Some(0xff); 4])], + 4, + ); + let params = VectorIndexParams::with_ivf_flat_params( + lance_linalg::distance::DistanceType::Hamming, + IvfBuildParams::try_with_centroids(2, Arc::new(centroids)).unwrap(), + ); + let mut dataset = source + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .as_ref() + .clone(); + dataset + .create_index(&["phash"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + let source = conn.open_table("images").execute().await.unwrap(); + + const BUCKETS: &str = "SELECT ivf_partition(phash) AS bucket, count(*) AS n, \ + min(id) AS lo, max(id) AS hi FROM images GROUP BY ivf_partition(phash)"; + let view = declare(&source, "buckets", BUCKETS).await.unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!( + rows(view.table(), &["bucket", "n", "lo", "hi"]).await, + ["0 10 0 9", "1 10 10 19"] + ); + } } diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 7061ac979..893c584e1 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -14,8 +14,8 @@ use reqwest::header::CONTENT_TYPE; use lance_namespace::models::{ CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest, - DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest, - ListNamespacesResponse, ListTablesRequest, ListTablesResponse, + DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, JsonArrowSchema, + ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest, ListTablesResponse, }; use crate::Error; @@ -36,6 +36,7 @@ use crate::secrets::SecretBinding; use crate::secrets::SecretInfo; use crate::table::BaseTable; use crate::utils::{reject_relative_segment, validate_table_name}; +use crate::view::ViewDescription; use super::client::{ ClientConfig, HeaderProvider, HttpSend, ID_DELIMITER, RequestResultExt, RestfulLanceDbClient, @@ -784,6 +785,59 @@ struct RemoteListedSecret { name: String, } +/// Define a view from a query. The name and its namespace are the path +/// identifier, so neither appears here. +#[derive(serde::Serialize)] +struct RemoteCreateViewRequest<'a> { + query: &'a str, +} + +/// What the service reports about one view. The schema arrives as the +/// namespace spec's JSON encoding, which is what `describe_table` uses too. +#[derive(serde::Deserialize)] +struct RemoteViewDescription { + name: String, + #[serde(default)] + namespace: Vec, + query: String, + default_database: String, + /// A path, like `namespace`: the root is the absent field rather than a + /// spelling of its own. + #[serde(default)] + default_namespace: Vec, + schema: JsonArrowSchema, +} + +impl RemoteViewDescription { + fn into_description(self, request_id: String) -> Result { + let schema = + lance_namespace::schema::convert_json_arrow_schema(&self.schema).map_err(|source| { + Error::Http { + source: format!("View '{}' has an undecodable schema: {source}", self.name) + .into(), + request_id, + status_code: None, + } + })?; + Ok(ViewDescription { + name: self.name, + namespace_path: self.namespace, + query: self.query, + default_database: self.default_database, + default_namespace_path: self.default_namespace, + schema: Arc::new(schema), + }) + } +} + +#[derive(serde::Deserialize)] +struct RemoteListViewsResponse { + #[serde(default)] + views: Vec, + #[serde(default)] + page_token: Option, +} + /// Bound on `list_jobs` page walking; a warning is logged when the listing /// is truncated at this many pages. const MAX_LIST_JOBS_PAGES: usize = 100; @@ -1005,6 +1059,10 @@ impl Database for RemoteDatabase { } async fn drop_function(&self, name: &str, version: &str) -> Result { + Ok(self.drop_function_async(name, version).await?.0) + } + + async fn drop_function_async(&self, name: &str, version: &str) -> Result<(bool, Job)> { let function_id = build_object_identifier("Function name", name, &[])?; let req = self .client @@ -1014,8 +1072,34 @@ impl Database for RemoteDatabase { })); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; - let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?; - Ok(response.dropped) + let status = response.status(); + let body = response.text().await.err_to_http(request_id.clone())?; + let dropped: RemoteDropFunctionResponse = + serde_json::from_str(&body).map_err(|source| Error::Http { + source: Box::new(source), + request_id: request_id.clone(), + status_code: Some(status), + })?; + let job = match status { + StatusCode::OK => Job::new_done(), + StatusCode::ACCEPTED => { + let job_id = extract_job_id(&body).ok_or_else(|| Error::Http { + source: "asynchronous Function drop response did not contain a valid job_id" + .into(), + request_id, + status_code: Some(status), + })?; + Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))) + } + _ => { + return Err(Error::Http { + source: "Function drop must return 200 OK or 202 Accepted".into(), + request_id, + status_code: Some(status), + }); + } + }; + Ok((dropped.dropped, job)) } async fn create_secret( @@ -1092,6 +1176,79 @@ impl Database for RemoteDatabase { response.json().await.err_to_http(request_id) } + async fn create_view( + &self, + name: &str, + query: &str, + namespace_path: &[String], + ) -> Result { + let view_id = build_object_identifier("View name", name, namespace_path)?; + let req = self + .client + .post(&format!("/v1/view/{view_id}/create")) + .json(&RemoteCreateViewRequest { query }); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let description: RemoteViewDescription = + response.json().await.err_to_http(request_id.clone())?; + description.into_description(request_id) + } + + async fn describe_view( + &self, + name: &str, + namespace_path: &[String], + ) -> Result { + let view_id = build_object_identifier("View name", name, namespace_path)?; + let req = self.client.post(&format!("/v1/view/{view_id}/describe")); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let description: RemoteViewDescription = + response.json().await.err_to_http(request_id.clone())?; + description.into_description(request_id) + } + + async fn drop_view(&self, name: &str, namespace_path: &[String]) -> Result<()> { + let view_id = build_object_identifier("View name", name, namespace_path)?; + let req = self.client.post(&format!("/v1/view/{view_id}/drop")); + let (request_id, response) = self.client.send(req).await?; + self.client.check_response(&request_id, response).await?; + Ok(()) + } + + async fn list_views(&self, namespace_path: &[String]) -> Result> { + let namespace_id = build_namespace_identifier(namespace_path)?; + let path = format!("/v1/namespace/{namespace_id}/view/list"); + let mut views = Vec::new(); + let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); + loop { + let mut req = self.client.get(&path); + if let Some(token) = &page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let response: RemoteListViewsResponse = + response.json().await.err_to_http(request_id.clone())?; + views.extend(response.views); + let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "View listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); + } + page_token = Some(next_page_token); + } + Ok(views) + } + async fn open_job(&self, job_id: &str) -> Result { let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string()); match crate::job::JobHandle::describe(&handle).await { @@ -1783,6 +1940,53 @@ mod tests { assert_eq!(job.id(), Some("j1-mv-drop")); } + #[tokio::test] + async fn test_drop_function_async_returns_job() { + let db = super::RemoteDatabase::new_mock(|request| { + assert_eq!(request.method(), "POST"); + assert_eq!(request.url().path(), "/v1/function/embed/drop"); + http::Response::builder() + .status(202) + .body(serde_json::json!({"dropped": true, "job_id": "j1-fn-drop"}).to_string()) + .unwrap() + }); + let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap(); + assert!(dropped); + assert_eq!(job.id(), Some("j1-fn-drop")); + } + + /// An unbound name and an inline deletion both answer `200`: the name is gone and + /// nothing is left to wait for, so the job is already finished. + #[tokio::test] + async fn test_drop_function_async_completed_inline() { + let db = super::RemoteDatabase::new_mock(|_| { + http::Response::builder() + .status(200) + .body(serde_json::json!({"dropped": false}).to_string()) + .unwrap() + }); + let (dropped, job) = db.drop_function_async("embed", "1").await.unwrap(); + assert!(!dropped); + assert_eq!(job.id(), None); + assert_eq!(job.status().await.unwrap(), "finished"); + job.wait().await.unwrap(); + } + + #[tokio::test] + async fn test_drop_function_async_rejects_incomplete_acceptance() { + for body in [ + r#"{"dropped":true}"#, + r#"{"dropped":true,"job_id":""}"#, + r#"{"dropped":true,"job_id":null}"#, + ] { + let db = super::RemoteDatabase::new_mock(move |_| { + http::Response::builder().status(202).body(body).unwrap() + }); + let error = db.drop_function_async("embed", "1").await.err().unwrap(); + assert!(error.to_string().contains("valid job_id")); + } + } + #[tokio::test] async fn test_drop_materialized_view_completed_inline() { let db = super::RemoteDatabase::new_mock(|request| { @@ -3721,6 +3925,166 @@ mod tests { .unwrap(); } + /// A view description carries the schema in the namespace spec's JSON + /// encoding, so the body a test asserts against is the one that encoding + /// produces rather than a hand-written guess at it. + fn view_description_body(name: &str, namespace: &[&str], query: &str) -> String { + let schema = Schema::new(vec![Field::new("id", DataType::Int32, true)]); + serde_json::json!({ + "name": name, + "namespace": namespace, + "query": query, + "default_database": "db", + "default_namespace": namespace, + "schema": lance_namespace::schema::arrow_schema_to_json(&schema).unwrap(), + }) + .to_string() + } + + #[tokio::test] + async fn test_create_view_posts_the_query_and_returns_the_resolved_schema() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::POST); + assert_eq!(request.url().path(), "/v1/view/analytics$adults/create"); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + // The name and its namespace address the view in the path, so the + // body says only what they cannot. + assert_eq!(body, serde_json::json!({"query": "SELECT id FROM people"})); + http::Response::builder() + .status(200) + .body(view_description_body( + "adults", + &["analytics"], + "SELECT id FROM people", + )) + .unwrap() + }); + let view = conn + .create_view("adults", "SELECT id FROM people", &["analytics".into()]) + .await + .unwrap(); + assert_eq!(view.name, "adults"); + assert_eq!(view.namespace_path, vec!["analytics".to_string()]); + assert_eq!(view.query, "SELECT id FROM people"); + assert_eq!(view.default_database, "db"); + assert_eq!(view.default_namespace_path, vec!["analytics".to_string()]); + assert_eq!(view.schema.field(0).name(), "id"); + } + + #[tokio::test] + async fn test_describe_and_drop_address_the_view_in_the_path() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/view/adults/describe"); + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder() + .status(200) + .body(view_description_body( + "adults", + &[], + "SELECT id FROM people", + )) + .unwrap() + }); + let view = conn.describe_view("adults", &[]).await.unwrap(); + assert!(view.namespace_path.is_empty()); + assert_eq!(view.schema.fields().len(), 1); + + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::POST); + assert_eq!(request.url().path(), "/v1/view/analytics$adults/drop"); + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_view("adults", &["analytics".into()]) + .await + .unwrap(); + } + + /// A schema the client cannot decode is a broken response, not a view + /// with no columns: reporting it as an error keeps a caller from reading + /// an empty schema as the truth about the view. + #[tokio::test] + async fn test_describe_view_rejects_an_undecodable_schema() { + let conn = Connection::new_with_handler(|_| { + http::Response::builder() + .status(200) + .body( + r#"{"name":"adults","query":"SELECT 1","default_database":"db", + "schema":{"fields":[{"name":"id","type":{"type":"nonesuch"}, + "nullable":true}]}}"#, + ) + .unwrap() + }); + let error = conn.describe_view("adults", &[]).await.unwrap_err(); + assert!(error.to_string().contains("undecodable schema"), "{error}"); + } + + #[tokio::test] + async fn test_list_views_walks_pages() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/analytics/view/list"); + let page = request + .url() + .query_pairs() + .find(|(key, _)| key == "page_token") + .map(|(_, value)| value.into_owned()); + let body = match page.as_deref() { + None => r#"{"views":[],"page_token":"p2"}"#, + Some("p2") => r#"{"views":["adults"]}"#, + Some(other) => panic!("unexpected page token: {other}"), + }; + http::Response::builder().status(200).body(body).unwrap() + }); + assert_eq!( + conn.list_views(&["analytics".into()]).await.unwrap(), + vec!["adults".to_string()] + ); + } + + /// A server that keeps handing back the same token would otherwise spin + /// forever. + #[tokio::test] + async fn test_list_views_rejects_a_repeated_page_token() { + let conn = Connection::new_with_handler(|_| { + http::Response::builder() + .status(200) + .body(r#"{"views":["adults"],"page_token":"same"}"#) + .unwrap() + }); + let error = conn.list_views(&[]).await.unwrap_err(); + assert!( + error.to_string().contains("repeated a page_token"), + "{error}" + ); + } + + /// A name carrying the delimiter would split back apart as a different + /// view, so it is refused before it reaches a route. + #[tokio::test] + async fn test_view_names_that_would_resplit_are_refused() { + let conn = Connection::new_with_handler(|_| -> http::Response { + panic!("an invalid identifier must not reach the service") + }); + for (name, namespace) in [ + ("analytics$adults", vec![]), + ("adults", vec!["ana$lytics".to_string()]), + ("", vec![]), + ("..", vec![]), + ] { + let error = match conn.describe_view(name, &namespace).await { + Ok(_) => panic!("accepted {name:?} in {namespace:?}"), + Err(error) => error.to_string(), + }; + // A view is not a table, so the refusal says so. + assert!( + error.contains("view name") || error.contains("view namespace path segment"), + "{error}" + ); + } + } + #[tokio::test] async fn test_create_function_async_sends_canonical_request_and_decodes_typed_job() { const REQUEST: &str = include_str!( diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 68c8409a9..6bbb49adf 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1486,6 +1486,20 @@ fn resolve_arrow_ipc_framing( }) } +/// An Arrow IPC stream carrying `schema` and no batches. +/// +/// The body of an all-null `add_columns`: the server reads the fields to +/// add straight off the schema message, so nothing but the field list is +/// worth sending. +fn write_ipc_schema(schema: &arrow_schema::Schema) -> Result> { + let mut body = Vec::new(); + { + let mut writer = arrow_ipc::writer::StreamWriter::try_new(&mut body, schema)?; + writer.finish()?; + } + Ok(body) +} + /// Strip media-type parameters before matching against the Arrow content types. fn base_media_type(content_type: &str) -> &str { match content_type.split_once(';') { @@ -2130,6 +2144,7 @@ impl BaseTable for RemoteTable { }) .collect(), filter: response.filter, + group_by: Vec::new(), limit: response.limit, }, }; @@ -3325,7 +3340,8 @@ impl BaseTable for RemoteTable { "schema evolution", crate::table::schema_evolution::new_column_names(&transforms), )?; - match transforms { + let path = format!("/v1/table/{}/add_columns/", self.identifier); + let request = match transforms { NewColumnTransform::SqlExpressions(expressions) => { let body = expressions .into_iter() @@ -3338,40 +3354,55 @@ impl BaseTable for RemoteTable { .collect::>(); let mut body = serde_json::json!({ "new_columns": body }); self.apply_branch_body(&mut body); - let request = self - .client - .post(&format!("/v1/table/{}/add_columns/", self.identifier)) - .json(&body); - let freshness_request = self.snapshot_freshness_headers(); - let (request_id, response) = self - .send_with_freshness(request, true, freshness_request) - .await?; - let response = self.check_table_response(&request_id, response).await?; - let body = response.text().await.err_to_http(request_id.clone())?; - - if body.trim().is_empty() { - // Backward compatible with old servers - return Ok(AddColumnsResult { version: 0 }); - } - - let result: AddColumnsResult = - serde_json::from_str(&body).map_err(|e| Error::Http { - source: format!("Failed to parse add_columns response: {}", e).into(), - request_id, - status_code: None, - })?; - - self.invalidate_schema_cache(); - self.track_write_version(freshness_request, result.version); - - Ok(result) + self.client.post(&path).json(&body) + } + // Every field becomes a column that reads null for existing + // rows. The schema goes over the wire as Arrow IPC rather than + // a JSON description of the types: that is what the server + // takes, and it is the only encoding that round-trips a field + // whole, including decimal precision and a timestamp's unit and + // timezone. With no JSON envelope the branch rides the query + // string, as it does for the other binary-bodied endpoints. + NewColumnTransform::AllNulls(schema) => { + let body = write_ipc_schema(schema.as_ref())?; + self.apply_branch_query( + self.client + .post(&path) + .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE) + .body(body), + ) } _ => { return Err(Error::NotSupported { - message: "Only SQL expressions are supported for adding columns".into(), + message: "Only SQL expressions and all-null column schemas are supported \ + for adding columns" + .into(), }); } + }; + + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; + let response = self.check_table_response(&request_id, response).await?; + let body = response.text().await.err_to_http(request_id.clone())?; + + if body.trim().is_empty() { + // Backward compatible with old servers + return Ok(AddColumnsResult { version: 0 }); } + + let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse add_columns response: {}", e).into(), + request_id, + status_code: None, + })?; + + self.invalidate_schema_cache(); + self.track_write_version(freshness_request, result.version); + + Ok(result) } async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result { @@ -7913,6 +7944,82 @@ mod tests { assert_eq!(result.version, if old_server { 0 } else { 43 }); } + /// A pyarrow schema goes over the wire as an Arrow IPC schema under an + /// Arrow content type, not as a JSON description of the types, so a + /// field arrives exactly as sent. The branch has no JSON envelope to + /// ride in and goes on the query string. + #[tokio::test] + async fn test_add_columns_sends_an_arrow_schema_for_all_nulls() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => simple_describe_response(), + "/v1/table/my_table/add_columns/" => { + assert_eq!(request.method(), "POST"); + assert_eq!( + request.headers().get("Content-Type").unwrap(), + ARROW_STREAM_CONTENT_TYPE + ); + assert_eq!( + request.url().query_pairs().find(|(k, _)| k == "branch"), + None, + "the main branch sends no branch parameter" + ); + + // The payload decodes back to the exact schema, types and + // field metadata included. + let body = request.body().unwrap().as_bytes().unwrap(); + let reader = + arrow_ipc::reader::StreamReader::try_new(std::io::Cursor::new(body), None) + .unwrap(); + let sent = reader.schema(); + assert_eq!(sent.fields().len(), 2); + assert_eq!(sent.field(0).name(), "ts"); + assert_eq!( + sent.field(0).data_type(), + &DataType::Timestamp( + arrow_schema::TimeUnit::Nanosecond, + Some("America/New_York".into()) + ) + ); + assert!(sent.field(0).is_nullable()); + assert_eq!(sent.field(1).name(), "price"); + assert_eq!(sent.field(1).data_type(), &DataType::Decimal128(38, 10)); + assert_eq!( + sent.field(1).metadata().get("unit").map(String::as_str), + Some("cents") + ); + + http::Response::builder() + .status(200) + .body(r#"{"version": 7}"#.to_string()) + .unwrap() + } + path => panic!("Unexpected request path: {path}"), + }); + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "ts", + DataType::Timestamp( + arrow_schema::TimeUnit::Nanosecond, + Some("America/New_York".into()), + ), + true, + ), + Field::new("price", DataType::Decimal128(38, 10), true).with_metadata( + std::collections::HashMap::from([("unit".to_string(), "cents".to_string())]), + ), + ])); + + let result = table + .add_columns() + .transform(NewColumnTransform::AllNulls(schema)) + .execute() + .await + .unwrap(); + + assert_eq!(result.version, 7); + } + /// A declaration is sent as `{name, computed}` for the server to plan; the /// client never types the expression itself. #[tokio::test] diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 23a870b42..679928cd3 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -418,23 +418,18 @@ pub(crate) fn ensure_not_function_bound>( touched: impl IntoIterator, ) -> Result<()> { ensure_supported_function_metadata(schema)?; - let mut protected = BTreeSet::new(); - for binding in function_bindings(schema)? { - for input in binding.inputs() { - protected.insert(field_root(&input.field_path)?); - } - protected.extend( - binding - .outputs() - .iter() - .map(|output| output.output_name.clone()), - ); - protected.extend( - binding - .assignment() - .map(|assignment| assignment.output_name.clone()), - ); - } + ensure_not_bound_by(&function_bindings(schema)?, operation, touched) +} + +/// [`ensure_not_function_bound`] against an explicit set of bindings, for a +/// caller that is retiring some of the schema's own in the same operation and +/// must be checked against what survives it. +pub(crate) fn ensure_not_bound_by>( + bindings: &[FunctionBinding], + operation: &str, + touched: impl IntoIterator, +) -> Result<()> { + let protected = protected_roots(bindings)?; if protected.is_empty() { return Ok(()); } @@ -452,6 +447,129 @@ pub(crate) fn ensure_not_function_bound>( Ok(()) } +/// Every column `bindings` depend on, inputs taken at their root. +fn protected_roots(bindings: &[FunctionBinding]) -> Result> { + let mut protected = BTreeSet::new(); + for binding in bindings { + for input in binding.inputs() { + protected.insert(field_root(&input.field_path)?); + } + protected.extend( + binding + .outputs() + .iter() + .map(|output| output.output_name.clone()), + ); + protected.extend( + binding + .assignment() + .map(|assignment| assignment.output_name.clone()), + ); + } + Ok(protected) +} + +/// What a drop must do besides removing columns, so that the Function +/// bindings it covers are retired rather than stranded. +#[derive(Debug, Default)] +pub struct FunctionUnbinding { + /// The bindings the drop leaves in place. Later columns in the same + /// operation are checked against these, not against the schema's. + pub retained: Vec, + /// Assignment columns of the retired bindings that the caller did not + /// name. Internal bookkeeping columns, so the drop adds them itself. + pub assignment_columns: Vec, + /// Columns whose `computed_column.*` declaration metadata the unbind + /// commit clears. + pub cleared_columns: Vec, + /// The new value of [`FUNCTION_BINDINGS_META_KEY`]; `None` deletes it. + pub bindings_metadata: Option, +} + +impl FunctionUnbinding { + /// True when the drop retires nothing and is an ordinary column drop. + pub fn is_noop(&self) -> bool { + self.cleared_columns.is_empty() + } + + /// Refuse `operation` on a path that a binding surviving this drop still + /// reads or writes. The retired ones no longer protect anything. + pub fn ensure_retained_unaffected>( + &self, + operation: &str, + touched: impl IntoIterator, + ) -> Result<()> { + ensure_not_bound_by(&self.retained, operation, touched) + } +} + +/// Plan the retirement of every Function binding whose outputs `columns` +/// covers. +/// +/// Naming one output of a binding means naming all of them: the outputs of +/// one Function are written by one refresh in one commit, so a binding that +/// kept some of them would have no coherent shape to write. A partial drop is +/// refused and names the siblings that are missing. +pub fn plan_function_unbinding( + schema: &ArrowSchema, + columns: &[&str], +) -> Result { + ensure_supported_function_metadata(schema)?; + let named = columns + .iter() + .map(|column| whole_column(column)) + .collect::>>>()? + .into_iter() + .flatten() + .collect::>(); + + let mut plan = FunctionUnbinding::default(); + for binding in function_bindings(schema)? { + let outputs = binding + .outputs() + .iter() + .map(|output| output.output_name.clone()) + .collect::>(); + let Some(dropped) = outputs.iter().find(|name| named.contains(name.as_str())) else { + plan.retained.push(binding); + continue; + }; + let missing = outputs + .iter() + .filter(|name| !named.contains(name.as_str())) + .cloned() + .collect::>(); + if !missing.is_empty() { + return Err(Error::InvalidInput { + message: format!( + "dropping Function output '{dropped}' must drop every output of its \ + binding; the same request must also name {}", + missing.join(", ") + ), + }); + } + plan.cleared_columns.extend(outputs); + if let Some(assignment) = binding.assignment() { + plan.cleared_columns.push(assignment.output_name.clone()); + if !named.contains(assignment.output_name.as_str()) { + plan.assignment_columns.push(assignment.output_name.clone()); + } + } + } + + // Re-encoding an untouched set would rewrite bytes a stricter reader + // compares against its own encoding, so leave the key alone when the drop + // retires nothing. + if !plan.is_noop() { + plan.bindings_metadata = if plan.retained.is_empty() { + None + } else { + Some(function_bindings_metadata(&plan.retained)?) + }; + } + Ok(plan) +} + /// Read a field's computed-column declaration, if it carries one. /// /// A field flagged computed but carrying no kind, or a SQL one missing its @@ -1320,7 +1438,30 @@ pub(crate) fn plan_function_application( /// Paths are compared at their root: a declaration reading `metadata` is /// invalidated by a change to `metadata.age` just as surely. pub(crate) fn ensure_not_an_input(schema: &SchemaRef, paths: &[&str]) -> Result<()> { + ensure_not_an_input_of(schema, paths, &[]) +} + +/// [`ensure_not_an_input`] where `removed` names columns whose own +/// declarations the same operation deletes. A reader that is going away +/// cannot be stranded by dropping what it reads, so a group drops together. +pub(crate) fn ensure_not_an_input_of( + schema: &SchemaRef, + paths: &[&str], + removed: &[&str], +) -> Result<()> { + // By identity, not spelling: a quoted path names the same column, while + // a path *into* one removes no declaration at all. + let removed = removed + .iter() + .map(|path| whole_column(path)) + .collect::>>()? + .into_iter() + .flatten() + .collect::>(); for declaration in computed_columns(schema) { + if removed.contains(&declaration.name) { + continue; + } // The expression, not stored inputs, is the source of truth; an // expression that no longer parses proves nothing, so refuse. let inputs = match &declaration.kind { @@ -1583,6 +1724,21 @@ pub(crate) fn field_root(path: &str) -> Result { }) } +/// The column a path names when it names the whole column, and `None` when it +/// addresses a field inside one. Retiring a binding takes the whole output; +/// a path into one is left to the ordinary guard to refuse. +fn whole_column(path: &str) -> Result> { + let mut segments = parse_field_path(path) + .map_err(|e| Error::InvalidInput { + message: format!("invalid column path '{path}': {e}"), + })? + .into_iter(); + let column = segments.next().ok_or_else(|| Error::InvalidInput { + message: format!("column path '{path}' is empty"), + })?; + Ok(segments.next().is_none().then_some(column)) +} + /// [`field_root`], falling back to the text before the first dot for a /// spelling lance would not resolve. pub(crate) fn root(path: &str) -> String { diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 796db09bd..4d0e2815d 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -9,7 +9,8 @@ //! - [`drop_columns`](execute_drop_columns): Remove columns from the table use arrow_schema::Schema as ArrowSchema; -use lance::dataset::{ColumnAlteration, NewColumnTransform}; +use lance::dataset::transaction::{Operation, Transaction, UpdateMap, UpdateMapEntry}; +use lance::dataset::{ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -240,14 +241,96 @@ pub(crate) async fn execute_drop_columns( table.dataset.ensure_mutable()?; let mut dataset = (*table.dataset.get().await?).clone(); let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema())); - computed_columns::ensure_not_function_bound(schema.as_ref(), "schema evolution", columns)?; - computed_columns::ensure_not_an_input(&schema, columns)?; - dataset.drop_columns(columns).await?; + let unbinding = computed_columns::plan_function_unbinding(schema.as_ref(), columns)?; + + let mut names = columns.iter().map(|c| c.to_string()).collect::>(); + names.extend(unbinding.assignment_columns.iter().cloned()); + let dropped = names.iter().map(String::as_str).collect::>(); + computed_columns::ensure_not_bound_by(&unbinding.retained, "schema evolution", &dropped)?; + computed_columns::ensure_not_an_input_of(&schema, &dropped, &dropped)?; + + if !unbinding.is_noop() { + // The retirement commits before the projection, so the whole + // projection has to be known valid against this revision first -- + // otherwise the binding is gone and the columns are not. Lance plans + // it rather than this repeating the rules: dropping a struct's last + // child removes the struct, and data files that disagree on metadata + // semantics are refused, neither of which an approximation here would + // get right. + dataset.plan_drop_columns(&dropped)?; + dataset = commit_function_unbinding(dataset, &unbinding).await?; + } + dataset.drop_columns(&dropped).await?; let version = dataset.version().version; table.dataset.update(dataset); Ok(DropColumnsResult { version }) } +/// Retire the bindings a drop covers, in the commit before it. +/// +/// A binding lives in three places -- the schema-metadata envelope, each +/// output's `computed_column.*` field metadata, and the columns themselves -- +/// and readers cross-check the first two, so a state with one of them removed +/// is a table that refuses every write. Clearing both metadata halves in one +/// `UpdateConfig` leaves the outputs as ordinary columns holding their last +/// values: valid on its own, and the drop that follows is then an ordinary +/// drop. Interrupted in between, the columns are still there to be dropped +/// again. +/// +/// Not folded into the drop's own `Operation::Project`, though it carries a +/// schema: the transaction proto records `Project` as fields alone, so the +/// metadata edit would survive only in this writer's memory. +async fn commit_function_unbinding( + dataset: Dataset, + unbinding: &computed_columns::FunctionUnbinding, +) -> Result { + let bindings = UpdateMap { + update_entries: vec![UpdateMapEntry { + key: computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(), + value: unbinding.bindings_metadata.clone(), + }], + replace: false, + }; + let mut field_metadata_updates = HashMap::new(); + for column in &unbinding.cleared_columns { + let field = dataset + .schema() + .field(column) + .ok_or_else(|| Error::InvalidInput { + message: format!("Function output '{column}' does not exist in the table"), + })?; + let cleared = field + .metadata + .keys() + .filter(|key| computed_columns::is_declaration_key(key)) + .map(|key| UpdateMapEntry { + key: key.clone(), + value: None, + }) + .collect::>(); + field_metadata_updates.insert( + field.id, + UpdateMap { + update_entries: cleared, + replace: false, + }, + ); + } + let transaction = Transaction::new( + dataset.manifest.version, + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: None, + schema_metadata_updates: Some(bindings), + field_metadata_updates, + }, + None, + ); + Ok(CommitBuilder::new(std::sync::Arc::new(dataset)) + .execute(transaction) + .await?) +} + /// Internal implementation of the update field metadata logic. /// /// Merges or replaces per-field metadata, addressing fields by dot-path. @@ -330,8 +413,9 @@ mod tests { use crate::query::{ExecutableQuery, QueryBase, Select}; use crate::table::NewColumnTransform; use crate::table::computed_columns::{ - FUNCTION_BINDINGS_META_KEY, ensure_supported_function_metadata, function_bindings, - function_bindings_metadata, function_computed_column_metadata, + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, FUNCTION_BINDINGS_META_KEY, + ensure_supported_function_metadata, function_bindings, function_bindings_metadata, + function_computed_column_metadata, is_declaration_key, }; use crate::{Error, Table}; use std::collections::HashMap; @@ -351,38 +435,71 @@ mod tests { ) .unwrap(); let table = conn.create_table("bound", batch).execute().await.unwrap(); - let binding = FunctionBinding::from_json(include_str!( + stamp_bindings(&table, &[text_features_binding("")]).await; + table + } + + /// The fixture binding, its id and output names suffixed so a table can + /// carry more than one. + fn text_features_binding(suffix: &str) -> FunctionBinding { + let raw = include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" - )) - .unwrap(); + ); + if suffix.is_empty() { + return FunctionBinding::from_json(raw).unwrap(); + } + FunctionBinding::from_json( + &raw.replace("fb_01K3TEXT", &format!("fb_01K3TEXT{suffix}")) + .replace("search_text", &format!("search_text{suffix}")) + .replace("search_token_count", &format!("search_token_count{suffix}")), + ) + .unwrap() + } + + /// Stamp bindings the way the server does it, since no local path + /// declares one: the envelope on the schema, a declaration marker on + /// every output field. + async fn stamp_bindings(table: &Table, bindings: &[FunctionBinding]) { let native = table.as_native().unwrap(); let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); dataset .update_schema_metadata(vec![( FUNCTION_BINDINGS_META_KEY.to_string(), - Some(function_bindings_metadata(std::slice::from_ref(&binding)).unwrap()), + Some(function_bindings_metadata(bindings).unwrap()), )]) .await .unwrap(); let inputs = ["title".to_string(), "body".to_string()]; - let outputs = binding - .outputs() + let outputs = bindings .iter() - .map(|output| { - ( - dataset.schema().field(&output.output_name).unwrap().id as u32, - function_computed_column_metadata( - binding.binding_id(), - output.output_ordinal, - &inputs, - ), - ) + .flat_map(|binding| { + let assignment = binding.assignment().map(|assignment| { + ( + assignment.output_name.clone(), + FUNCTION_ASSIGNMENT_OUTPUT_ORDINAL, + ) + }); + binding + .outputs() + .iter() + .map(|output| (output.output_name.clone(), output.output_ordinal)) + .chain(assignment) + .map(|(name, ordinal)| { + ( + dataset.schema().field(&name).unwrap().id as u32, + function_computed_column_metadata( + binding.binding_id(), + ordinal, + &inputs, + ), + ) + }) + .collect::>() }) .collect::>(); dataset.replace_field_metadata(outputs).await.unwrap(); native.dataset.update(dataset); ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); - table } fn metadata_update(path: &str) -> FieldMetadataUpdate { @@ -438,6 +555,18 @@ mod tests { ); } + fn incomplete_group(err: Error) { + assert!( + matches!(&err, Error::InvalidInput { message } + if message.contains("must drop every output of its binding")), + "{err:?}" + ); + } + + async fn bindings_of(table: &Table) -> Vec { + function_bindings(table.schema().await.unwrap().as_ref()).unwrap() + } + /// Every schema-evolution door refuses a column a binding reads or /// writes, including a rename onto one. #[tokio::test] @@ -445,7 +574,14 @@ mod tests { let table = bound_table().await; let version = table.version().await.unwrap(); for column in ["title", "body", "search_text", "search_token_count"] { - bound(table.drop_columns(&[column]).await.unwrap_err()); + // A lone drop of an output is refused too, but for its siblings + // rather than for the binding; see the retirement tests below. + let err = table.drop_columns(&[column]).await.unwrap_err(); + if column.starts_with("search_") { + incomplete_group(err); + } else { + bound(err); + } bound( table .alter_columns(&[ColumnAlteration::new(column.into()).rename("moved".into())]) @@ -500,7 +636,12 @@ mod tests { let table = bound_table().await; let version = table.version().await.unwrap(); for path in ["`title`", "`search_text`", "`title`.nested"] { - bound(table.drop_columns(&[path]).await.unwrap_err()); + let err = table.drop_columns(&[path]).await.unwrap_err(); + if path == "`search_text`" { + incomplete_group(err); + } else { + bound(err); + } bound( table .alter_columns(&[ColumnAlteration::new(path.into()).set_nullable(false)]) @@ -1038,6 +1179,264 @@ mod tests { assert_eq!(drop_result.version, v4); } + // Retiring a Function binding by dropping its outputs (ENT-2669). + + /// The whole output set goes, and the binding with it: no envelope, no + /// declaration markers, and the inputs it held are ordinary again. + #[tokio::test] + async fn dropping_every_output_retires_the_binding() { + let table = bound_table().await; + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + + let schema = table.schema().await.unwrap(); + assert!(schema.field_with_name("search_text").is_err()); + assert!(schema.field_with_name("search_token_count").is_err()); + assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + assert!(bindings_of(&table).await.is_empty()); + + // What the binding used to protect is now ordinary schema. + table.drop_columns(&["title"]).await.unwrap(); + table + .update_field_metadata(&[metadata_update("body")]) + .await + .unwrap(); + ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); + } + + /// One output of a multi-output binding cannot go alone: its siblings are + /// written by the same refresh in the same commit. + #[tokio::test] + async fn dropping_part_of_an_output_group_names_the_missing_siblings() { + let table = bound_table().await; + let version = table.version().await.unwrap(); + for (named, missing) in [ + ("search_text", "search_token_count"), + ("search_token_count", "search_text"), + ] { + let err = table.drop_columns(&[named]).await.unwrap_err(); + let Error::InvalidInput { message } = &err else { + panic!("{err:?}"); + }; + assert!( + message.contains("must drop every output of its binding") + && message.contains(missing), + "{message}" + ); + } + assert_eq!(table.version().await.unwrap(), version); + assert_eq!(bindings_of(&table).await.len(), 1); + } + + /// An input is protected by the binding, not by its own name, so it drops + /// in the request that retires the binding reading it. + #[tokio::test] + async fn an_input_drops_with_the_binding_that_reads_it() { + let table = bound_table().await; + bound(table.drop_columns(&["title"]).await.unwrap_err()); + + table + .drop_columns(&["title", "body", "search_text", "search_token_count"]) + .await + .unwrap(); + let schema = table.schema().await.unwrap(); + assert_eq!(schema.fields().len(), 1); + assert!(schema.field_with_name("spare").is_ok()); + assert!(bindings_of(&table).await.is_empty()); + } + + /// Lance resolves a quoted spelling to the same field, so it retires the + /// same binding rather than slipping past the group rule. + #[tokio::test] + async fn a_quoted_output_spelling_retires_the_same_binding() { + let table = bound_table().await; + incomplete_group(table.drop_columns(&["`search_text`"]).await.unwrap_err()); + table + .drop_columns(&["`search_text`", "`search_token_count`"]) + .await + .unwrap(); + assert!(bindings_of(&table).await.is_empty()); + } + + /// The unbind lands before the drop, and on its own: at that version the + /// outputs are still there as plain columns, with no binding and no + /// declaration metadata left pointing at one. A crash in between leaves + /// that state, which every later write must accept. + #[tokio::test] + async fn the_unbind_commit_leaves_a_valid_table_before_the_drop() { + let table = bound_table().await; + let before = table.version().await.unwrap(); + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + assert_eq!(table.version().await.unwrap(), before + 2); + + table.checkout(before + 1).await.unwrap(); + let schema = table.schema().await.unwrap(); + assert!(!schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + for column in ["search_text", "search_token_count"] { + let field = schema.field_with_name(column).unwrap(); + assert!( + !field.metadata().keys().any(|key| is_declaration_key(key)), + "{column}: {:?}", + field.metadata() + ); + } + ensure_supported_function_metadata(&schema).unwrap(); + assert!(function_bindings(&schema).unwrap().is_empty()); + } + + /// A table carrying two bindings retires only the one whose outputs the + /// drop names; the other keeps its envelope entry and its protection. + #[tokio::test] + async fn retiring_one_binding_leaves_the_others_standing() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + ("search_text_2", Utf8, ["a b"]), + ("search_token_count_2", Int64, [2]) + ) + .unwrap(); + let table = conn.create_table("two", batch).execute().await.unwrap(); + stamp_bindings( + &table, + &[text_features_binding(""), text_features_binding("_2")], + ) + .await; + + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + let remaining = bindings_of(&table).await; + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].binding_id(), "fb_01K3TEXT_2"); + // The survivor still protects the inputs it shared with the retired one. + bound(table.drop_columns(&["title"]).await.unwrap_err()); + // And its own outputs are still a group. + incomplete_group(table.drop_columns(&["search_text_2"]).await.unwrap_err()); + } + + /// A multi-output binding also owns a hidden `__function_assignment_*` + /// column, which the caller never names. Retiring the binding has to take + /// it: leaving it behind orphans a column nothing can fill or explain. + #[tokio::test] + async fn retiring_a_binding_drops_its_assignment_column() { + const ASSIGNMENT: &str = "__function_assignment_fb_01K3TEXT"; + + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + (ASSIGNMENT, Boolean, [Some(true)]), + ("spare", Int32, [1]) + ) + .unwrap(); + let table = conn + .create_table("assigned", batch) + .execute() + .await + .unwrap(); + + let raw = include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + ) + .replace( + r#""future_binding""#, + &format!( + r#""assignment": {{"output_name": "{ASSIGNMENT}", "output_field_id": -1}}, "future_binding""# + ), + ) + // The assignment column is a physical sibling, so it belongs to the + // binding's output schema too -- the server writes it that way. + .replace( + r#"{"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}"#, + &format!( + r#"{{"name": "search_token_count", "nullable": true, "type": {{"type": "int64"}}}}, {{"name": "{ASSIGNMENT}", "nullable": true, "type": {{"type": "bool"}}}}"# + ), + ); + let binding = FunctionBinding::from_json(&raw).unwrap(); + assert!(binding.assignment().is_some(), "fixture must carry one"); + stamp_bindings(&table, std::slice::from_ref(&binding)).await; + + // The assignment column is the binding's, so it is protected too... + bound(table.drop_columns(&[ASSIGNMENT]).await.unwrap_err()); + + // ...and goes with the outputs without the caller naming it. + table + .drop_columns(&["search_text", "search_token_count"]) + .await + .unwrap(); + let schema = table.schema().await.unwrap(); + assert!( + schema.field_with_name(ASSIGNMENT).is_err(), + "the assignment column outlived its binding: {:?}", + schema.fields().iter().map(|f| f.name()).collect::>() + ); + assert!(bindings_of(&table).await.is_empty()); + ensure_supported_function_metadata(&schema).unwrap(); + } + + /// An input error must not cost the binding. The retirement commits + /// first, so a projection that could never succeed is refused before it, + /// leaving the table exactly as it was. + #[rstest::rstest] + #[case::unknown_column( + &["search_text", "search_token_count", "nope"], + "does not exist" + )] + #[case::every_column( + &["title", "body", "search_text", "search_token_count", "spare"], + "Cannot drop all columns" + )] + #[case::every_column_quoted( + &["`title`", "`body`", "`search_text`", "`search_token_count`", "`spare`"], + "Cannot drop all columns" + )] + #[tokio::test] + async fn an_invalid_drop_leaves_the_binding_in_place( + #[case] columns: &[&str], + #[case] expected: &str, + ) { + let table = bound_table().await; + let version = table.version().await.unwrap(); + + let err = table.drop_columns(columns).await.unwrap_err(); + let Error::InvalidInput { message } = &err else { + panic!("{err:?}"); + }; + assert!(message.contains(expected), "{message}"); + + // Read durable state, not this handle: the error path never calls + // `dataset.update`, so the cached handle would report the old version + // even if the unbind had committed. + table.checkout_latest().await.unwrap(); + assert_eq!(table.version().await.unwrap(), version); + let schema = table.schema().await.unwrap(); + assert!(schema.metadata().contains_key(FUNCTION_BINDINGS_META_KEY)); + assert_eq!(bindings_of(&table).await.len(), 1); + for column in ["search_text", "search_token_count"] { + assert!( + schema + .field_with_name(column) + .unwrap() + .metadata() + .keys() + .any(|key| is_declaration_key(key)), + "{column} lost its declaration metadata" + ); + } + ensure_supported_function_metadata(&schema).unwrap(); + } + #[tokio::test] async fn test_update_field_metadata() { let conn = connect("memory://").execute().await.unwrap(); diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index f80cc876a..52b7bd5e8 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -174,6 +174,27 @@ pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Resul validate_secret_component("Secret name", name) } +/// Validate a view name and every segment of the namespace path holding it. +/// +/// Worded for a view rather than deferring to [`validate_table_name`]: a view +/// is not a table, and a caller who mistypes one should not be told their +/// table name is invalid. The rule is the same one every object name follows, +/// which is what lets the `$`-joined identifier split back apart. +pub fn validate_view_reference(name: &str, namespace_path: &[String]) -> Result<()> { + for segment in namespace_path { + validate_view_component("view namespace path segment", segment)?; + } + validate_view_component("view name", name) +} + +/// Validate one component of a view identifier: a view name, or one segment of +/// the namespace path holding it. +pub fn validate_view_component(what: &str, value: &str) -> Result<()> { + check_object_name(value).map_err(|reason| Error::InvalidInput { + message: format!("invalid {what} '{value}': {reason}"), + }) +} + /// Validate all components of a namespace /// /// Iterates through all namespace components and validates each one. diff --git a/rust/lancedb/src/view.rs b/rust/lancedb/src/view.rs new file mode 100644 index 000000000..ca9703209 --- /dev/null +++ b/rust/lancedb/src/view.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Views: a named query a database stores and plans on every read. +//! +//! A view holds no rows. What it stores is the statement that defines it and +//! the schema that statement resolved to, so a reader sees the sources as they +//! are now rather than as they were when the view was created. That is the +//! whole difference from [`crate::materialized_view`], which holds rows and +//! moves them forward by refresh. +//! +//! The verbs live on [`crate::connection::Connection`]. A view is read through +//! SQL, by name, so there is no `open_view` returning a +//! [`crate::table::Table`]: there would be no rows behind it. + +use arrow_schema::SchemaRef; + +/// What a database records about one view. +/// +/// Returned by [`crate::connection::Connection::describe_view`], and by +/// `create_view` so a caller has the resolved schema without a second call. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ViewDescription { + /// The view's name within its namespace. + pub name: String, + /// The namespace holding the view; empty is the root namespace. + pub namespace_path: Vec, + /// The defining query, as the database stores it. + pub query: String, + /// The database that unqualified table names in `query` resolve against. + /// A view created from a session connected elsewhere can name a database + /// other than its own, so this is part of what the query means rather + /// than a restatement of where the view lives. + pub default_database: String, + /// The namespace path those unqualified names resolve against; empty is + /// the root namespace. + /// + /// Recorded with the view because it outlives the session that declared + /// it: a reader that resolved the query against its own default namespace + /// could read a different table than the view was defined over. + pub default_namespace_path: Vec, + /// The schema the defining query resolved to when the view was created. + pub schema: SchemaRef, +}