Compare commits

..

1 Commits

Author SHA1 Message Date
Lance Release 8c68e0c619 Bump version: 0.38.0-beta.16 → 0.38.0 2026-08-31 07:38:30 +00:00
21 changed files with 279 additions and 837 deletions
Generated
+92 -71
View File
@@ -535,9 +535,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.92"
version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
@@ -1443,9 +1443,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
[[package]]
name = "bytemuck"
version = "1.25.2"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
dependencies = [
"bytemuck_derive",
]
@@ -3455,8 +3455,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f727719438dfdb74f358a347c91ff81b6e7084a6421f34de3e473ce271f10caa"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4816,9 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be781f40c7a75f9eae2188a2f71174acb7a360dca97163db40b041d0828dea48"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4890,9 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb97fd9875f3036d7c2561aa5b16eb87b80ccabaa4eeb5e6099b19cc662f1cd8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4914,8 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,17 +4929,20 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f"
dependencies = [
"arrow-array",
"arrow-schema",
"half",
"lance-arrow-scalar",
]
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f45658c5b2dc9aada41b66ee44b83af3fa888b7385ae414bae951b12a9f1cd3"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4952,9 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27af3df3a7d08897efccd04461df31cedf0880c4b86a055ddce48e423d27f967"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4991,9 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c377f837df5296e92f9fad724c83c1bef4e74d5af6e5a9312e9307e1dead8614"
dependencies = [
"arrow",
"arrow-array",
@@ -5013,8 +5022,9 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e1a5065fa4bc184e36e32681f10f8f4680ad8cedc9377b4c088dce8c5b8da"
dependencies = [
"arrow",
"arrow-array",
@@ -5031,8 +5041,9 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13e5e95e0fd3d74f7938f4bee623041421b323b5c61f242c8622a1f48a202527"
dependencies = [
"proc-macro2",
"quote",
@@ -5041,8 +5052,9 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1625653c55c65f3426e281f6e29b54c603f38a40bd4cebd707bd4f3ea48be6c5"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,8 +5087,9 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7e13c9266b478fc98f36ee19347c4658f7a6613fed77778b1a455fe1b88552e"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5107,8 +5120,9 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0e0cb95f2c4f341c4dd04ac60f6a89ea26a6f75e09570225cbda4854c8b088e"
dependencies = [
"arc-swap",
"arrow",
@@ -5172,8 +5186,9 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79ccd371977c1f7168da259d66ad37154f23146f093d46136bc7f79559f00f2c"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5195,8 +5210,9 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "414d50997391b1ac83dc183c1612fdff88f58b959806078dc4c5e465154566de"
dependencies = [
"arrow",
"arrow-array",
@@ -5236,8 +5252,9 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ff55b152ef23a56d7ba7e4d1b2c9cf0cc79aef6ee607c115597557ea4059f41"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5251,8 +5268,9 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09991c13ab282b731e323619613914e08da9cc82b312f904e58c128b23f2f0e3"
dependencies = [
"arrow",
"async-trait",
@@ -5264,8 +5282,9 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ec0bc005f6bb8f120774eb4a9ba02e10463d8338167a46cbb1391d46680a174"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5318,8 +5337,9 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8f676a2a1837cc85b77feb5326d3296827da964e40d67144f646563302a6ce9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5333,8 +5353,9 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd33054347395048b1d842dfb85a13f7801392c2da5f39425db62f00a481744b"
dependencies = [
"arrow",
"arrow-array",
@@ -5374,8 +5395,9 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc9ad9ae24f045dfddd538a39e55a28fa7e1ca6ad9f23e20d4087eaf2bb66f7"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5388,8 +5410,9 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.5#556637791d0048c2b4f1342dd84b67c8bbd65259"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3bfa6f0164c8b7056150f5682ce4d415a335b59b04c479873fda04b200117d27"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5402,7 +5425,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0"
version = "0.38.0-beta.15"
dependencies = [
"ahash",
"anyhow",
@@ -5490,7 +5513,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0"
version = "0.38.0-beta.15"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5515,7 +5538,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0"
version = "0.38.0-beta.15"
dependencies = [
"arrow",
"async-trait",
@@ -5748,9 +5771,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.34"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "loom"
@@ -6001,9 +6024,9 @@ dependencies = [
[[package]]
name = "moka"
version = "0.12.16"
version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
dependencies = [
"async-lock",
"crossbeam-channel",
@@ -6097,15 +6120,14 @@ dependencies = [
[[package]]
name = "napi"
version = "3.12.2"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
dependencies = [
"bitflags 2.11.1",
"chrono",
"ctor 1.0.12",
"futures",
"libc",
"napi-build",
"napi-sys",
"nohash-hasher",
@@ -6117,15 +6139,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
[[package]]
name = "napi-derive"
version = "3.6.3"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
dependencies = [
"convert_case",
"ctor 1.0.12",
@@ -6137,9 +6159,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.2"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
dependencies = [
"convert_case",
"proc-macro2",
@@ -8602,9 +8624,9 @@ dependencies = [
[[package]]
name = "roaring"
version = "0.11.5"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
dependencies = [
"bytemuck",
"byteorder",
@@ -9064,9 +9086,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
dependencies = [
"base64 0.22.1",
"bs58",
@@ -9074,7 +9096,6 @@ dependencies = [
"hex",
"indexmap 1.9.3",
"indexmap 2.14.0",
"jiff",
"schemars 0.9.0",
"schemars 1.2.1",
"serde_core",
@@ -9085,9 +9106,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
dependencies = [
"darling 0.23.0",
"proc-macro2",
@@ -10454,9 +10475,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.26.0"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
dependencies = [
"getrandom 0.4.2",
"js-sys",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.5", "tag" = "v12.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0", default-features = false }
lance-core = "=11.0.0"
lance-datagen = "=11.0.0"
lance-file = "=11.0.0"
lance-io = { "version" = "=11.0.0", default-features = false }
lance-index = "=11.0.0"
lance-linalg = "=11.0.0"
lance-namespace = "=11.0.0"
lance-namespace-impls = { "version" = "=11.0.0", default-features = false }
lance-table = "=11.0.0"
lance-testing = "=11.0.0"
lance-datafusion = "=11.0.0"
lance-encoding = "=11.0.0"
lance-arrow = "=11.0.0"
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.5</lance-core.version>
<lance-core.version>11.0.0</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -5
View File
@@ -664,11 +664,7 @@ impl JsFullTextQuery {
}
fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
// `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
// path, which is only usable from generated `#[napi]` argument conversion. This is a
// manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
// unwraps the class without requiring a borrow scope.
if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
Ok(FullTextSearchQuery::new_query(query.inner.clone()))
} else if let Ok(Some(query_text)) = query.get::<String>("query") {
let mut query_text = query_text;
-1
View File
@@ -150,7 +150,6 @@ class Connection(object):
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
async def cancel_job(self, job_id: str) -> bool: ...
-18
View File
@@ -712,16 +712,6 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog.
Returns True when the version changed to Dropped and False for an
idempotent replay. Local connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
@@ -1423,10 +1413,6 @@ class LanceDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
@@ -2257,10 +2243,6 @@ class AsyncConnection(object):
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
async def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog."""
return await self._inner.drop_function(name, version)
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
+10 -97
View File
@@ -521,12 +521,6 @@ class RefreshColumnResult(_RemoteValue):
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
_NESTED_BLOB_COLLECTION_ERROR = (
"unsupported Arrow type for Function signature: Blob v2 fields nested under "
"collection types are not supported"
)
_GRAMMAR_PRIMITIVES = (
@@ -597,19 +591,6 @@ def _validate_exact_arrow_field(field: pa.Field) -> None:
"unsupported Arrow type for Function signature: lance.blob.v2 "
f"requires a supported Blob storage layout, got {field}"
)
metadata = {
(key.decode() if isinstance(key, bytes) else key): (
value.decode() if isinstance(value, bytes) else value
)
for key, value in (field.metadata or {}).items()
}
if metadata and metadata != {
_ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME
}:
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
"field metadata must contain only its canonical extension marker"
)
elif field.metadata:
raise TypeError(
"unsupported Arrow type for Function signature: field metadata "
@@ -674,84 +655,23 @@ def _canonical_arrow_field(field: pa.Field) -> str:
return _canonical_arrow_type(field.type)
def _blob_storage_type(field: pa.Field) -> pa.DataType:
data_type = field.type
if isinstance(data_type, pa.ExtensionType):
return data_type.storage_type
return data_type
def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
storage = _blob_storage_type(field)
if not pa.types.is_struct(storage):
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
"requires struct storage"
)
return {
"type": "struct",
"fields": [
{
"name": child.name,
"nullable": child.nullable,
"type": (
{"type": "large_binary"}
if pa.types.is_large_binary(child.type)
else _exact_arrow_type(child.type)
),
}
for child in storage
],
}
def _data_type_has_blob_v2(data_type: pa.DataType) -> bool:
if pa.types.is_struct(data_type):
return any(
_is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
for field in data_type
)
if (
pa.types.is_list(data_type)
or pa.types.is_large_list(data_type)
or pa.types.is_fixed_size_list(data_type)
):
field = data_type.value_field
return _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
if pa.types.is_map(data_type):
return any(
_is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
for field in (data_type.key_field, data_type.item_field)
)
return False
def _exact_arrow_field(
field: pa.Field, *, inside_collection: bool = False
) -> dict[str, Any]:
def _exact_arrow_field(field: pa.Field) -> dict[str, Any]:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
if inside_collection:
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
return {
"name": field.name,
"nullable": field.nullable,
"type": _exact_blob_storage_type(field),
"metadata": {
_ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME,
},
}
raise TypeError(
"unsupported Arrow type for Function signature: nested Blob v2 "
"fields are not supported; declare Blob parameters or named result "
"fields directly"
)
value = {
"name": field.name,
"nullable": field.nullable,
"type": _exact_arrow_type(field.type, inside_collection=inside_collection),
"type": _exact_arrow_type(field.type),
}
return value
def _exact_arrow_type(
data_type: pa.DataType, *, inside_collection: bool = False
) -> dict[str, Any]:
def _exact_arrow_type(data_type: pa.DataType) -> dict[str, Any]:
for candidate, name in _GRAMMAR_PRIMITIVES:
if data_type == candidate:
return {"type": name}
@@ -765,10 +685,7 @@ def _exact_arrow_type(
)
return {
"type": "struct",
"fields": [
_exact_arrow_field(field, inside_collection=inside_collection)
for field in fields
],
"fields": [_exact_arrow_field(field) for field in fields],
}
if (
pa.types.is_list(data_type)
@@ -793,15 +710,11 @@ def _exact_arrow_type(
if pa.types.is_large_list(data_type)
else "fixed_size_list"
),
"fields": [
_exact_arrow_field(data_type.value_field, inside_collection=True)
],
"fields": [_exact_arrow_field(data_type.value_field)],
}
if pa.types.is_fixed_size_list(data_type):
value["length"] = data_type.list_size
return value
if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
-4
View File
@@ -749,10 +749,6 @@ class RemoteDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
@@ -12,8 +12,6 @@ from datetime import date
import http.server
import json
from pathlib import Path
import subprocess
import sys
import threading
from typing import Optional
@@ -69,80 +67,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
}
def _main_udf_source(
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
) -> str:
return (
"from __future__ import annotations\n"
"from lancedb.functions import udf\n"
f"THRESHOLD = {threshold}\n"
"\n"
"@udf\n"
f"def label(value: {input_annotation}) -> str:\n"
f" return 'big' if value {comparison} THRESHOLD else 'small'\n"
"\n"
"assert label.__module__ == '__main__'\n"
"print(label.registration_request.to_canonical_json())\n"
)
def _run_main_udf(path: Path, source: str) -> dict:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source)
result = subprocess.run(
[sys.executable, str(path)],
check=True,
capture_output=True,
text=True,
)
return json.loads(result.stdout)
def test_main_udf_registration_identity_is_stable_across_processes_and_paths(
tmp_path,
):
source = _main_udf_source()
original_path = tmp_path / "original" / "job.py"
moved_path = tmp_path / "moved" / "renamed_job.py"
original_runs = [_run_main_udf(original_path, source) for _ in range(2)]
moved_run = _run_main_udf(moved_path, source)
assert len({run["artifact"]["digest"] for run in [*original_runs, moved_run]}) == 1
assert all(
run["signature"] == original_runs[0]["signature"]
for run in [original_runs[1], moved_run]
)
assert original_runs[0] == original_runs[1] == moved_run
body_change = _run_main_udf(
tmp_path / "changes" / "body.py", _main_udf_source(comparison=">")
)
global_change = _run_main_udf(
tmp_path / "changes" / "global.py", _main_udf_source(threshold=21)
)
annotation_change = _run_main_udf(
tmp_path / "changes" / "annotation.py",
_main_udf_source(input_annotation="float"),
)
baseline = original_runs[0]
assert baseline["signature"] == body_change["signature"]
assert baseline["signature"] == global_change["signature"]
assert baseline["signature"] != annotation_change["signature"]
assert (
len(
{
baseline["artifact"]["digest"],
body_change["artifact"]["digest"],
global_change["artifact"]["digest"],
annotation_change["artifact"]["digest"],
}
)
== 4
)
def _run_packaged(definition, *args):
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
source = base64.b64decode(definition.registration_request.artifact.content.data)
@@ -668,167 +592,6 @@ def test_blob_fields_use_the_scalar_function_semantic_type():
assert signature.output.arrow_type == "blob_v2"
def test_whole_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=pa.field(
"payload",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
lancedb.blob("image", nullable=False),
]
),
nullable=False,
),
)
def inspect_blob(image):
return {"mime_type": "image/png", "image": image}
output = inspect_blob.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.arrow_type) for field in output.fields] == [
("mime_type", "utf8"),
("image", "blob_v2"),
]
def test_struct_blob_signature_fields_preserve_exact_metadata_and_nullability():
nested_input = pa.field(
"payload",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
pa.field(
"nested",
pa.struct([lancedb.blob("image", nullable=True)]),
nullable=True,
),
]
),
nullable=True,
)
nested_output = pa.field(
"result",
pa.struct(
[
pa.field("mime_type", pa.string(), nullable=False),
pa.field(
"nested",
pa.struct([lancedb.blob("image", nullable=True)]),
nullable=False,
),
]
),
nullable=False,
)
@udf(input_schema=pa.schema([nested_input]), output_schema=nested_output)
def copy_payload(payload):
return payload
signature = copy_payload.registration_request.signature
input_type = json.loads(signature.inputs[0].arrow_type)
assert input_type["fields"][1]["nullable"] is True
input_blob = input_type["fields"][1]["type"]["fields"][0]
assert input_blob["nullable"] is True
assert input_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
assert signature.output.kind == "named_struct"
nested_result = next(
field for field in signature.output.fields if field.name == "nested"
)
output_type = json.loads(nested_result.arrow_type)
output_blob = output_type["fields"][0]
assert output_blob["nullable"] is True
assert output_blob["metadata"] == {"ARROW:extension:name": "lance.blob.v2"}
def test_struct_blob_signature_supports_multiple_struct_levels():
recursive = pa.field(
"value",
pa.struct(
[
pa.field(
"level_1",
pa.struct(
[
pa.field(
"level_2",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
]
),
nullable=False,
)
]
),
nullable=False,
)
@udf(
input_schema=pa.schema([recursive]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["level_1"]["level_2"]["image"])
encoded = json.loads(blob_size.registration_request.signature.inputs[0].arrow_type)
blob = encoded["fields"][0]["type"]["fields"][0]["type"]["fields"][0]
assert blob["metadata"]["ARROW:extension:name"] == "lance.blob.v2"
@pytest.mark.parametrize(
"data_type",
[
pa.list_(lancedb.blob("item", nullable=False)),
pa.large_list(lancedb.blob("item", nullable=False)),
pa.list_(lancedb.blob("item", nullable=False), 2),
pa.map_(pa.string(), lancedb.blob("value", nullable=False).type),
],
)
def test_blob_signature_rejects_collection_ancestors(data_type):
with pytest.raises(
TypeError,
match="Blob v2 fields nested under collection types are not supported",
):
@udf(
input_schema=pa.schema([pa.field("value", data_type, nullable=False)]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value)
def test_blob_signature_rejects_collection_below_a_struct():
nested = pa.field(
"value",
pa.struct(
[
pa.field(
"images",
pa.list_(lancedb.blob("item", nullable=False)),
nullable=False,
)
]
),
nullable=False,
)
with pytest.raises(
TypeError,
match="Blob v2 fields nested under collection types are not supported",
):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["images"])
def test_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
@@ -890,6 +653,22 @@ def test_blob_marker_rejects_invalid_storage_layout():
return len(image)
def test_nested_blob_signature_field_has_a_clear_error():
nested = pa.field(
"value",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="nested Blob v2 fields are not supported"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["image"])
def test_nested_non_blob_extension_is_not_silently_unwrapped():
class TestExtension(pa.ExtensionType):
def __init__(self):
@@ -1162,8 +941,6 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
db.create_function_async(normalize_score)
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version="fv_exact")
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version="fv_exact")
@contextlib.contextmanager
@@ -1209,12 +986,6 @@ def _mock_remote_function_catalog():
"version": "fv_exact",
}
response = state["version"]
elif self.path == "/v1/functions/drop":
assert body == {
"name": "normalize_score",
"version": "fv_exact",
}
response = {"dropped": True}
else:
status = 404
response = {"error": "not found"}
@@ -1273,40 +1044,3 @@ def test_blocking_remote_registration_returns_function_version():
"/v1/functions/create",
"/v1/jobs/describe",
]
def test_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
@pytest.mark.asyncio
async def test_async_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert await db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
-11
View File
@@ -629,17 +629,6 @@ impl Connection {
})
}
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
})
}
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
-15
View File
@@ -523,21 +523,6 @@ impl Connection {
.await
}
/// Drop one exact immutable Function version from the remote catalog.
///
/// Returns `true` when the server appended a Dropped transition and
/// `false` for an idempotent replay. Local databases return
/// [`Error::NotSupported`].
pub async fn drop_function(
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
) -> Result<bool> {
self.internal
.drop_function(name.as_ref(), version.as_ref())
.await
}
/// Rename a table in the database.
///
/// This is only supported in LanceDB Cloud.
-4
View File
@@ -307,10 +307,6 @@ pub trait Database:
) -> Result<crate::function::FunctionVersion> {
function_catalog_not_supported()
}
/// Drop one exact immutable Function version from the remote catalog.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
/// waiting on or cancelling the job. The handle is constructed without a
/// server round trip; an unknown id surfaces when the handle is used.
+120 -38
View File
@@ -13,7 +13,7 @@ use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
use lance_datafusion::utils::StreamingWriteSource;
use lance_file::version::LanceFileVersion;
use lance_io::object_store::{ReadDirOptions, StorageOptionsAccessor, StorageOptionsProvider};
use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
use lance_table::io::commit::commit_handler_from_url;
use object_store::local::LocalFileSystem;
use snafu::ResultExt;
@@ -282,14 +282,11 @@ impl std::fmt::Display for ListingDatabase {
const LANCE_EXTENSION: &str = "lance";
/// The table a listed child of the database names, or `None` if the child is not a table.
/// The table a listed child directory holds, or `None` if it is not a table at all.
///
/// A table is the directory `<name>.lance`; a loose file or any other directory under the
/// database prefix belongs to something else. `dir_suffix` is `.lance`, built once by the
/// caller rather than per child.
/// The table a listed child directory holds, or `None` if it is not a table at all.
///
/// Only directories are considered, so a loose object named like a table is not one.
fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<String> {
location
.filename()?
@@ -297,6 +294,75 @@ fn table_name(location: &object_store::path::Path, dir_suffix: &str) -> Option<S
.map(String::from)
.filter(|name| !name.is_empty())
}
/// One page of the table directories under the database directory, in key order.
struct DirPage {
/// The table directories the page holds, as the store lists them.
common_prefixes: Vec<object_store::path::Path>,
/// Resumes after this page, or `None` when the page reached the end of the level.
page_token: Option<String>,
}
/// Where a listed location sits inside the database directory — the space page tokens live
/// in — or `None` if it is not a child of that directory at all. Matching both halves of the
/// prefix drops a location that merely starts with the directory's name (`dbx/y` against
/// `db/`) as well as the marker object some stores keep for the directory itself.
fn relative_key<'a>(prefix: Option<&str>, location: &'a str) -> Option<&'a str> {
let relative = match prefix {
Some(prefix) => location.strip_prefix(prefix)?,
None => location,
};
(!relative.is_empty()).then_some(relative)
}
/// One page of the table directories under `base_path`, one directory level deep.
///
/// Lance 11 exposes no paginated directory listing, so the level is listed in full and paged
/// locally: table directories go into key order (a directory's key keeps its trailing `/`,
/// so a token is never a table name), the page is the smallest `limit` of them past
/// `page_token`, and the token handed back is the key of the last directory the page took —
/// so a page that took nothing ends the listing rather than resuming from a position no page
/// ever reached. Only `<name>.lance/` directories enter the page: loose objects, other
/// directories, and a bare `.lance/` never take a page slot or name a token, which keeps a
/// page to exactly one listing of the level. Correct on every store, at the cost of that one
/// full-level listing per page.
async fn read_dir_page(
object_store: &ObjectStore,
base_path: &object_store::path::Path,
page_token: Option<String>,
limit: Option<usize>,
) -> Result<DirPage> {
let listed = object_store.list_with_delimiter(Some(base_path)).await?;
let prefix = {
let base = base_path.as_ref();
(!base.is_empty()).then(|| format!("{base}/"))
};
let table_dir_suffix = format!(".{LANCE_EXTENSION}/");
let mut children: Vec<(String, object_store::path::Path)> = listed
.common_prefixes
.into_iter()
.filter_map(|location| {
let key = format!("{}/", relative_key(prefix.as_deref(), location.as_ref())?);
(key.len() > table_dir_suffix.len() && key.ends_with(&table_dir_suffix))
.then_some((key, location))
})
.collect();
children.sort_unstable_by(|(left, _), (right, _)| left.cmp(right));
if let Some(resume) = &page_token {
children.retain(|(key, _)| key > resume);
}
let total = children.len();
children.truncate(limit.unwrap_or(total).min(total));
let page_token = match children.last() {
Some((last, _)) if children.len() < total => Some(last.clone()),
_ => None,
};
Ok(DirPage {
common_prefixes: children.into_iter().map(|(_, location)| location).collect(),
page_token,
})
}
const ENGINE: &str = "engine";
const MIRRORED_STORE: &str = "mirroredStore";
@@ -982,8 +1048,7 @@ impl Database for ListingDatabase {
let mut tables = Vec::new();
let mut page_token = request.page_token.filter(|token| !token.is_empty());
// A page of nothing: the store rejects a limit of zero, and no table was handed over
// for a token to resume after.
// A page of nothing: no table was handed over for a token to resume after.
if limit == Some(0) {
return Ok(ListTablesResponse {
context: None,
@@ -992,35 +1057,21 @@ impl Database for ListingDatabase {
});
}
loop {
// Ask only for what the page still has room for, so a database holding more
// than one page costs one request per page rather than one per table.
let listing = self
.object_store
.read_dir_page(
self.base_path.clone(),
ReadDirOptions {
page_token: page_token.take(),
limit: limit.map(|limit| limit - tables.len()),
},
)
.await?;
page_token = listing.page_token;
// Only child directories can be tables, and the store already separates them
// out, so the objects in the page are not looked at.
tables.extend(
listing
.result
.common_prefixes
.iter()
.filter_map(|location| table_name(location, &dir_suffix)),
);
// Children that are not tables leave the page short of the limit, so keep
// going until the page is full or the database runs out.
if page_token.is_none() || limit.is_none_or(|limit| tables.len() >= limit) {
break;
}
}
// The page holds only table directories, so one call — and the one full-level
// listing behind it — fills it.
let page = read_dir_page(
&self.object_store,
&self.base_path,
page_token.take(),
limit,
)
.await?;
page_token = page.page_token;
tables.extend(
page.common_prefixes
.iter()
.filter_map(|location| table_name(location, &dir_suffix)),
);
Ok(ListTablesResponse {
context: None,
@@ -1666,8 +1717,8 @@ mod tests {
}
/// Only directories named `<name>.lance` are tables; loose files and other directories
/// under the database prefix are not. A page spent on them is filled from the next one,
/// so a page holding only non-tables does not read as an empty database.
/// under the database prefix are not. They never take a page slot, so even a `limit`
/// smaller than the clutter ahead of the first table returns that table.
#[tokio::test]
async fn test_listing_ignores_non_table_children() {
let (tempdir, db) = setup_database().await;
@@ -1686,6 +1737,37 @@ mod tests {
assert_eq!(page.tables, vec!["real"]);
}
/// The Lance 11 fallback pages locally over one full-level listing, so a bounded page
/// costs exactly one listing call — clutter ahead of the first table must not buy extra
/// round trips.
#[tokio::test]
async fn test_one_full_listing_per_public_page() {
use crate::io::object_store::io_tracking::IoStatsHolder;
use lance_io::object_store::WrappingObjectStore;
let (tempdir, mut db) = setup_database().await;
create_tables(&db, &["real"]).await;
std::fs::write(tempdir.path().join("aaa-loose.lance"), b"not a table").unwrap();
create_dir_all(tempdir.path().join("aaa-scratch")).unwrap();
let io_stats = IoStatsHolder::default();
let mut tracked_store = (*db.object_store).clone();
tracked_store.inner =
io_stats.wrap(&tracked_store.store_prefix, tracked_store.inner.clone());
db.object_store = Arc::new(tracked_store);
let page = db
.list_tables(ListTablesRequest {
limit: Some(1),
..Default::default()
})
.await
.unwrap();
assert_eq!(page.tables, vec!["real"]);
assert_eq!(io_stats.incremental_stats().read_iops, 1);
}
#[tokio::test]
async fn listing_ignores_empty_table_name() {
let (tempdir, db) = setup_database().await;
+3 -10
View File
@@ -539,7 +539,9 @@ impl Database for LanceNamespaceDatabase {
self.namespace
.drop_table(drop_request)
.await
.map_err(|e| map_namespace_lance_error(e, name))?;
.map_err(|e| Error::Runtime {
message: format!("Failed to drop table: {}", e),
})?;
Ok(())
}
@@ -1493,15 +1495,6 @@ mod tests {
.expect("Failed to list tables");
assert!(!table_names_after.contains(&"drop_test".to_string()));
let error = conn
.drop_table("drop_test", &["test_ns".into()])
.await
.expect_err("dropping a missing table should fail");
assert!(
matches!(error, Error::TableNotFound { ref name, .. } if name == "drop_test"),
"expected TableNotFound, got: {error:?}"
);
// Verify: Cannot open dropped table
let open_result = conn.open_table("drop_test").execute().await;
assert!(open_result.is_err());
+1 -9
View File
@@ -10,7 +10,7 @@ use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
UploadPart, list::PaginatedListStore, path::Path,
UploadPart, path::Path,
};
use async_trait::async_trait;
@@ -187,14 +187,6 @@ impl WrappingObjectStore for MirroringObjectStoreWrapper {
secondary: self.secondary.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn PaginatedListStore>,
) -> Option<Arc<dyn PaginatedListStore>> {
Some(original)
}
}
// windows pathing can't be simply concatenated
@@ -12,7 +12,7 @@ use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
UploadPart, list::PaginatedListStore, path::Path,
UploadPart, path::Path,
};
#[derive(Debug, Default)]
@@ -57,14 +57,6 @@ impl WrappingObjectStore for IoStatsHolder {
stats: self.0.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn PaginatedListStore>,
) -> Option<Arc<dyn PaginatedListStore>> {
Some(original)
}
}
impl IoTrackingStore {
-38
View File
@@ -533,11 +533,6 @@ struct RemoteListJobsResponse {
page_token: Option<String>,
}
#[derive(serde::Deserialize)]
struct RemoteDropFunctionResponse {
dropped: bool,
}
/// Bound on `list_jobs` page walking; a warning is logged when the listing
/// is truncated at this many pages.
const MAX_LIST_JOBS_PAGES: usize = 100;
@@ -588,20 +583,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
response.json().await.err_to_http(request_id)
}
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
let req = self
.client
.post("/v1/functions/drop")
.json(&serde_json::json!({
"name": name,
"version": version,
}));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?;
Ok(response.dropped)
}
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
self.client.clone(),
@@ -2708,25 +2689,6 @@ mod tests {
assert_eq!(version.version(), "fv_01K3EXACT");
}
#[tokio::test]
async fn test_drop_function_sends_exact_version_and_decodes_replay() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/drop");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
body,
serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"})
);
http::Response::builder()
.status(200)
.body(r#"{"dropped":false}"#)
.unwrap()
});
assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap());
}
#[tokio::test]
async fn test_conn_job_waits_to_done() {
let polls = Arc::new(AtomicUsize::new(0));
-16
View File
@@ -4183,14 +4183,6 @@ mod tests {
parent_list_calls: self.parent_list_calls.clone(),
})
}
fn wrap_paginated(
&self,
_store_prefix: &str,
_original: Arc<dyn object_store::list::PaginatedListStore>,
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
None
}
}
#[tokio::test]
@@ -4294,14 +4286,6 @@ mod tests {
self.called.store(true, Ordering::Relaxed);
original
}
fn wrap_paginated(
&self,
_store_prefix: &str,
original: Arc<dyn object_store::list::PaginatedListStore>,
) -> Option<Arc<dyn object_store::list::PaginatedListStore>> {
Some(original)
}
}
#[tokio::test]
+18 -188
View File
@@ -589,13 +589,16 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
.and_then(|metadata| metadata.get(ARROW_EXT_NAME_KEY))
.map(String::as_str)
== Some(BLOB_V2_EXT_NAME);
if is_blob_v2 || field.r#type.fields.is_some() {
if is_blob_v2 {
let arrow_field = lance_namespace::schema::convert_json_arrow_field(field)
.map_err(|e| invalid_function(format!("invalid Function input field: {e}")))?;
validate_function_blob_nesting(&arrow_field, false)?;
if is_blob_v2 {
return Ok(FUNCTION_BLOB_V2_TYPE.to_string());
if !has_supported_blob_v2_layout(&arrow_field) {
return Err(invalid_function(format!(
"Function input '{}' has an invalid Blob v2 storage layout",
arrow_field.name()
)));
}
return Ok(FUNCTION_BLOB_V2_TYPE.to_string());
}
if field.r#type.fields.is_none() && field.r#type.length.is_none() {
Ok(field.r#type.r#type.clone())
@@ -614,34 +617,6 @@ fn has_supported_blob_v2_layout(field: &ArrowField) -> bool {
)
}
fn validate_function_blob_nesting(field: &ArrowField, inside_collection: bool) -> Result<()> {
if field.is_blob_v2() {
if inside_collection {
return Err(invalid_function(format!(
"Function field '{}' nests Blob v2 under a collection, which Function signatures do not support",
field.name()
)));
}
if !has_supported_blob_v2_layout(field) {
return Err(invalid_function(format!(
"Function field '{}' has an invalid Blob v2 storage layout",
field.name()
)));
}
return Ok(());
}
match field.data_type() {
DataType::Struct(fields) => fields
.iter()
.try_for_each(|field| validate_function_blob_nesting(field, inside_collection)),
DataType::List(field)
| DataType::LargeList(field)
| DataType::FixedSizeList(field, _)
| DataType::Map(field, _) => validate_function_blob_nesting(field, true),
_ => Ok(()),
}
}
/// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
/// any nested `<...>`.
fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
@@ -722,22 +697,21 @@ fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
}
fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonArrowField> {
let field = if raw == FUNCTION_BLOB_V2_TYPE {
lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![crate::blob(
name, nullable,
)]))
if raw == FUNCTION_BLOB_V2_TYPE {
return lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
crate::blob(name, nullable),
]))
.map_err(|e| invalid_function(format!("could not encode Blob v2 output field: {e}")))?
.fields
.into_iter()
.next()
.ok_or_else(|| invalid_function("Blob v2 output field is missing"))?
} else {
JsonArrowField::new(name.to_string(), nullable, parse_output_arrow_type(raw)?)
};
let arrow_field = lance_namespace::schema::convert_json_arrow_field(&field)
.map_err(|e| invalid_function(format!("invalid Function output field: {e}")))?;
validate_function_blob_nesting(&arrow_field, false)?;
Ok(field)
.ok_or_else(|| invalid_function("Blob v2 output field is missing"));
}
Ok(JsonArrowField::new(
name.to_string(),
nullable,
parse_output_arrow_type(raw)?,
))
}
fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
@@ -2745,28 +2719,6 @@ mod tests {
.unwrap()
}
fn exact_arrow_type(field: ArrowField) -> String {
let json =
lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![field])).unwrap();
serde_json::to_string(json.fields[0].r#type.as_ref()).unwrap()
}
fn single_input_application(path: &str) -> FunctionApplication {
FunctionApplication::from_json(
&serde_json::json!({
"function": {"name": "inspect", "version": "fv_nested_blob"},
"inputs": [{
"parameter": "value",
"kind": "column",
"value": {"path": path}
}],
"output": {"kind": "scalar", "arrow_type": "int64", "nullable": false}
})
.to_string(),
)
.unwrap()
}
fn binding_from_plan(plan: &FunctionDeclarationPlan) -> FunctionBinding {
let inputs = plan
.input_bindings
@@ -3206,128 +3158,6 @@ mod tests {
assert_eq!(fields[1].data_type(), &DataType::Int32);
}
#[test]
fn test_struct_blob_input_preserves_exact_schema_and_nullability() {
let payload = ArrowField::new(
"payload",
DataType::Struct(Fields::from(vec![
ArrowField::new("mime_type", DataType::Utf8, false),
ArrowField::new(
"nested",
DataType::Struct(Fields::from(vec![crate::blob("image", true)])),
true,
),
])),
true,
);
let plan = plan_function_application(
&ArrowSchema::new(vec![payload]),
&single_input_application("payload"),
Some("size"),
)
.unwrap();
let declared: JsonArrowDataType =
serde_json::from_str(&plan.input_bindings[0].arrow_type).unwrap();
let DataType::Struct(fields) =
lance_namespace::schema::convert_json_arrow_type(&declared).unwrap()
else {
panic!("expected a struct Function input")
};
assert!(fields[1].is_nullable());
let DataType::Struct(nested) = fields[1].data_type() else {
panic!("expected a recursive struct Function input")
};
assert!(nested[0].is_blob_v2());
assert!(nested[0].is_nullable());
let exact = lance_namespace::schema::convert_json_arrow_schema(&plan.input_schema).unwrap();
let DataType::Struct(fields) = exact.field(0).data_type() else {
panic!("expected exact input schema to retain the struct")
};
let DataType::Struct(nested) = fields[1].data_type() else {
panic!("expected exact input schema to retain the nested struct")
};
assert!(nested[0].is_blob_v2());
}
#[test]
fn test_recursive_blob_result_plans_one_whole_named_struct_column() {
let details_type = exact_arrow_type(ArrowField::new(
"details",
DataType::Struct(Fields::from(vec![crate::blob("image", true)])),
false,
));
let application = FunctionApplication::from_json(
&serde_json::json!({
"function": {"name": "inspect", "version": "fv_nested_blob"},
"inputs": [],
"output": {
"kind": "named_struct",
"fields": [
{"name": "mime_type", "arrow_type": "utf8", "nullable": false},
{"name": "details", "arrow_type": details_type, "nullable": false}
]
}
})
.to_string(),
)
.unwrap();
let plan = plan_function_application(&ArrowSchema::empty(), &application, Some("payload"))
.unwrap();
assert_eq!(plan.outputs.len(), 1);
assert_eq!(plan.outputs[0].result_field, WHOLE_RESULT_FIELD);
let schema =
lance_namespace::schema::convert_json_arrow_schema(&plan.output_schema).unwrap();
assert_eq!(schema.field(0).name(), "payload");
let DataType::Struct(fields) = schema.field(0).data_type() else {
panic!("whole named result must be one struct column")
};
assert_eq!(
fields.iter().map(|field| field.name()).collect::<Vec<_>>(),
["mime_type", "details"]
);
let DataType::Struct(details) = fields[1].data_type() else {
panic!("expected recursive result struct")
};
assert!(details[0].is_blob_v2());
assert!(!fields.iter().any(|field| field.name() == "payload"));
}
#[test]
fn test_blob_children_under_collections_are_rejected() {
let collections = vec![
DataType::List(Arc::new(crate::blob("item", false))),
DataType::LargeList(Arc::new(crate::blob("item", false))),
DataType::FixedSizeList(Arc::new(crate::blob("item", false)), 2),
DataType::Map(
Arc::new(ArrowField::new(
"entries",
DataType::Struct(Fields::from(vec![
ArrowField::new("key", DataType::Utf8, false),
crate::blob("value", false),
])),
false,
)),
false,
),
];
for data_type in collections {
let schema = ArrowSchema::new(vec![ArrowField::new("value", data_type, false)]);
let error = plan_function_application(
&schema,
&single_input_application("value"),
Some("size"),
)
.unwrap_err();
assert!(
error.to_string().contains("under a collection"),
"got: {error}"
);
}
}
#[test]
fn test_blob_whole_struct_binding_accepts_full_logical_layout() {
let input = crate::blob("image", false);
+1 -1
View File
@@ -300,7 +300,7 @@ async fn build_read_context(
for shard_id in shard_ids {
let manifest_store =
ShardManifestStore::new(store.clone(), &base_path, shard_id, scan_batch_size);
if let Some(manifest) = manifest_store.latest().await? {
if let Some(manifest) = manifest_store.read_latest().await? {
snapshots.push(snapshot_from_manifest(shard_id, &manifest, &exclude));
}
}
@@ -45,11 +45,7 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
.get_function("normalize_score", "fv_exact")
.await
.unwrap_err();
let drop_error = connection
.drop_function("normalize_score", "fv_exact")
.await
.unwrap_err();
for error in [create_error, lookup_error, drop_error] {
for error in [create_error, lookup_error] {
assert!(matches!(
error,
Error::NotSupported { message }