Compare commits

..

1 Commits

Author SHA1 Message Date
lancedb automation 110e3024e3 chore: update lance dependency to v9.0.0-rc.2 2026-07-21 17:27:41 +00:00
42 changed files with 254 additions and 744 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.32.0-beta.3"
current_version = "0.32.0-beta.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Generated
+156 -181
View File
@@ -157,9 +157,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.104"
version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
[[package]]
name = "approx"
@@ -535,13 +535,13 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.91"
version = "0.1.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
"syn 2.0.117",
]
[[package]]
@@ -1750,7 +1750,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -2288,9 +2288,9 @@ dependencies = [
[[package]]
name = "datafusion"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098"
checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040"
dependencies = [
"arrow",
"arrow-schema",
@@ -2335,9 +2335,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99"
checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139"
dependencies = [
"arrow",
"async-trait",
@@ -2360,9 +2360,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02"
checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891"
dependencies = [
"arrow",
"async-trait",
@@ -2383,9 +2383,9 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9"
checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2407,9 +2407,9 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2"
checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1"
dependencies = [
"futures",
"log",
@@ -2418,9 +2418,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd"
checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe"
dependencies = [
"arrow",
"async-trait",
@@ -2448,9 +2448,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9"
checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2472,9 +2472,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7"
checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada"
dependencies = [
"arrow",
"async-trait",
@@ -2495,9 +2495,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba"
checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f"
dependencies = [
"arrow",
"async-trait",
@@ -2518,15 +2518,15 @@ dependencies = [
[[package]]
name = "datafusion-doc"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891"
checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e"
[[package]]
name = "datafusion-execution"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6"
checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2546,9 +2546,9 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3"
checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0"
dependencies = [
"arrow",
"arrow-schema",
@@ -2568,9 +2568,9 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356"
checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2"
dependencies = [
"arrow",
"datafusion-common",
@@ -2580,9 +2580,9 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7"
checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2612,9 +2612,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b"
checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09"
dependencies = [
"arrow",
"datafusion-common",
@@ -2633,9 +2633,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545"
checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c"
dependencies = [
"arrow",
"datafusion-common",
@@ -2645,9 +2645,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f"
checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5"
dependencies = [
"arrow",
"arrow-ord",
@@ -2670,9 +2670,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383"
checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26"
dependencies = [
"arrow",
"async-trait",
@@ -2686,9 +2686,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d"
checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78"
dependencies = [
"arrow",
"datafusion-common",
@@ -2703,9 +2703,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441"
checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2713,9 +2713,9 @@ dependencies = [
[[package]]
name = "datafusion-macros"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5"
dependencies = [
"datafusion-doc",
"quote",
@@ -2724,9 +2724,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179"
checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281"
dependencies = [
"arrow",
"chrono",
@@ -2743,9 +2743,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0"
checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c"
dependencies = [
"arrow",
"datafusion-common",
@@ -2764,9 +2764,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859"
checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235"
dependencies = [
"arrow",
"datafusion-common",
@@ -2779,9 +2779,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8"
checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8"
dependencies = [
"arrow",
"chrono",
@@ -2796,9 +2796,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183"
checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36"
dependencies = [
"arrow",
"datafusion-common",
@@ -2814,9 +2814,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1"
checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc"
dependencies = [
"arrow",
"arrow-data",
@@ -2847,9 +2847,9 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7"
checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3"
dependencies = [
"arrow",
"datafusion-common",
@@ -2863,9 +2863,9 @@ dependencies = [
[[package]]
name = "datafusion-session"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a"
checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417"
dependencies = [
"async-trait",
"datafusion-common",
@@ -2877,9 +2877,9 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "54.1.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69"
checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099"
dependencies = [
"arrow",
"bigdecimal",
@@ -3006,7 +3006,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3229,7 +3229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4218,7 +4218,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.3",
"socket2 0.5.10",
"system-configuration",
"tokio",
"tower-service",
@@ -4517,7 +4517,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -4611,7 +4611,7 @@ dependencies = [
"portable-atomic-util",
"serde_core",
"wasm-bindgen",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arc-swap",
"arrow",
@@ -4852,8 +4852,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrayref",
"crunchy",
@@ -4909,8 +4909,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4933,7 +4933,6 @@ dependencies = [
"object_store",
"pin-project",
"prost",
"quick_cache",
"rand 0.9.5",
"roaring",
"serde_json",
@@ -4949,8 +4948,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow",
"arrow-array",
@@ -4980,8 +4979,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow",
"arrow-array",
@@ -4998,8 +4997,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"proc-macro2",
"quote",
@@ -5008,8 +5007,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5044,8 +5043,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,14 +5074,13 @@ dependencies = [
[[package]]
name = "lance-index"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arc-swap",
"arrow",
"arrow-arith",
"arrow-array",
"arrow-ipc",
"arrow-ord",
"arrow-schema",
"arrow-select",
@@ -5143,8 +5141,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5166,8 +5164,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow",
"arrow-arith",
@@ -5210,8 +5208,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5227,8 +5225,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow",
"async-trait",
@@ -5240,8 +5238,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5295,8 +5293,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5311,8 +5309,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow",
"arrow-array",
@@ -5351,8 +5349,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5365,8 +5363,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "10.0.0-beta.5"
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
version = "9.0.0-rc.2"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-rc.2#e4650e99bcce0a311df2fd4cc826103d2b0ca51b"
dependencies = [
"icu_segmenter",
"jieba-rs",
@@ -5379,7 +5377,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.32.0-beta.3"
version = "0.32.0-beta.2"
dependencies = [
"ahash",
"anyhow",
@@ -5467,7 +5465,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.32.0-beta.3"
version = "0.32.0-beta.2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5492,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.35.0-beta.3"
version = "0.35.0-beta.2"
dependencies = [
"arrow",
"async-trait",
@@ -5592,9 +5590,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.189"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libloading"
@@ -6065,9 +6063,9 @@ dependencies = [
[[package]]
name = "napi"
version = "3.11.0"
version = "3.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
checksum = "6826e5ddc15589b2d68c8ad5321c18e85d40488e93e32962f362e572669bccf6"
dependencies = [
"bitflags 2.11.1",
"chrono",
@@ -6090,9 +6088,9 @@ checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"
[[package]]
name = "napi-derive"
version = "3.6.0"
version = "3.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a49c513341a61a16a10af6efcce46b30d0822ba2d4fb197d24d33dfc199c78d5"
checksum = "b0fe526e81c105d3640516fcde83909dd1afe757c0d7a15af58830b5bc0fb9a1"
dependencies = [
"convert_case",
"ctor 1.0.5",
@@ -6104,9 +6102,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.0.0"
version = "5.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4747005fa3e2c9989ac45a723a514c5db2411238b72981a3cda4c701a9dfea17"
checksum = "514281397bcddd9ea9a876c7a21a57bff2374237a000ca9a64ea0211ec1993e2"
dependencies = [
"convert_case",
"proc-macro2",
@@ -6117,9 +6115,9 @@ dependencies = [
[[package]]
name = "napi-sys"
version = "3.3.0"
version = "3.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85fbf1fa9f1babfe396d74bbbf52b3643770243e8f5b0b46715d4caf7f0dfc9a"
checksum = "73e43cf2eb0bd1bf95a43c07c076ebd2da5d1e015a71c3d201faeffffcc0ecac"
dependencies = [
"libloading",
]
@@ -6208,7 +6206,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -7587,8 +7585,8 @@ version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck 0.5.0",
"itertools 0.14.0",
"heck 0.4.1",
"itertools 0.11.0",
"log",
"multimap",
"petgraph",
@@ -7607,7 +7605,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.11.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -7803,18 +7801,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "quick_cache"
version = "0.6.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
dependencies = [
"ahash",
"equivalent",
"hashbrown 0.16.1",
"parking_lot",
]
[[package]]
name = "quinn"
version = "0.11.9"
@@ -7828,7 +7814,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls 0.23.40",
"socket2 0.6.3",
"socket2 0.5.10",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -7866,9 +7852,9 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.3",
"socket2 0.5.10",
"tracing",
"windows-sys 0.60.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -8174,9 +8160,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.13.1"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2"
dependencies = [
"aho-corasick",
"memchr",
@@ -8186,9 +8172,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.16"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
@@ -8642,7 +8628,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -8713,7 +8699,7 @@ dependencies = [
"security-framework",
"security-framework-sys",
"webpki-root-certs",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -8918,9 +8904,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "serde"
version = "1.0.229"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
@@ -8928,29 +8914,29 @@ dependencies = [
[[package]]
name = "serde_core"
version = "1.0.229"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.229"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.3",
"syn 2.0.117",
]
[[package]]
name = "serde_json"
version = "1.0.151"
version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
"itoa",
"memchr",
@@ -9281,7 +9267,7 @@ version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9293,7 +9279,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
dependencies = [
"heck 0.5.0",
"heck 0.4.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -9609,17 +9595,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
@@ -9737,7 +9712,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -9940,9 +9915,9 @@ dependencies = [
[[package]]
name = "tokio"
version = "1.53.1"
version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"bytes",
"libc",
@@ -10427,9 +10402,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.24.0"
version = "1.23.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a"
dependencies = [
"getrandom 0.4.2",
"js-sys",
@@ -10714,7 +10689,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=9.0.0-rc.2", default-features = false, "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.0.0-rc.2", default-features = false, "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.0.0-rc.2", default-features = false, "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.0.0-rc.2", "tag" = "v9.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
+30 -165
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.32.0-beta.3</version>
<version>0.32.0-beta.2</version>
</dependency>
```
@@ -249,57 +249,6 @@ try (BufferAllocator allocator = new RootAllocator();
}
```
### Creating an Empty Table
To create an empty table, send an Arrow IPC stream that contains the table schema and no record batches.
The schema in the IPC stream becomes the table schema, and rows can be inserted later.
```java
import org.lance.namespace.model.CreateTableRequest;
import org.lance.namespace.model.CreateTableResponse;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import java.io.ByteArrayOutputStream;
import java.nio.channels.Channels;
import java.util.Arrays;
Schema schema = new Schema(Arrays.asList(
new Field("id", FieldType.nullable(new ArrowType.Int(32, true)), null),
new Field("name", FieldType.nullable(new ArrowType.Utf8()), null),
new Field("embedding",
FieldType.nullable(new ArrowType.FixedSizeList(128)),
Arrays.asList(new Field("item",
FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
null)))
));
byte[] emptyTableData;
try (BufferAllocator allocator = new RootAllocator();
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
root.setRowCount(0);
ByteArrayOutputStream out = new ByteArrayOutputStream();
try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(out))) {
writer.start();
writer.end();
}
emptyTableData = out.toByteArray();
}
CreateTableRequest request = new CreateTableRequest();
request.setId(Arrays.asList("my_namespace", "empty_table"));
CreateTableResponse response = namespaceClient.createTable(request, emptyTableData);
```
### Insert
```java
@@ -482,88 +431,9 @@ query.setVector(vector);
byte[] result = namespaceClient.queryTable(query);
```
## Indexing
### Reading Query Results
The Java SDK exposes the REST namespace index operations through the same `LanceNamespace` client.
Index creation runs asynchronously, so use `listTableIndices` or `describeTableIndexStats` to check progress.
### Creating a Vector Index
```java
import org.lance.namespace.model.CreateTableIndexRequest;
import org.lance.namespace.model.CreateTableIndexResponse;
CreateTableIndexRequest request = new CreateTableIndexRequest();
request.setId(Arrays.asList("my_namespace", "my_table"));
request.setColumn("embedding");
request.setIndexType("IVF_PQ");
request.setDistanceType("cosine");
request.setName("embedding_idx");
CreateTableIndexResponse response = namespaceClient.createTableIndex(request);
System.out.println("Index transaction: " + response.getTransactionId());
```
### Creating a Scalar Index
```java
import org.lance.namespace.model.CreateTableIndexRequest;
import org.lance.namespace.model.CreateTableScalarIndexResponse;
CreateTableIndexRequest request = new CreateTableIndexRequest();
request.setId(Arrays.asList("my_namespace", "my_table"));
request.setColumn("category");
request.setIndexType("BTREE");
request.setName("category_idx");
CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
System.out.println("Index transaction: " + response.getTransactionId());
```
### Creating a Full Text Search Index
```java
import org.lance.namespace.model.CreateTableIndexRequest;
import org.lance.namespace.model.CreateTableScalarIndexResponse;
CreateTableIndexRequest request = new CreateTableIndexRequest();
request.setId(Arrays.asList("my_namespace", "my_table"));
request.setColumn("text_column");
request.setIndexType("FTS");
request.setName("text_idx");
request.setBaseTokenizer("simple");
request.setLowerCase(true);
request.setWithPosition(true);
CreateTableScalarIndexResponse response = namespaceClient.createTableScalarIndex(request);
System.out.println("Index transaction: " + response.getTransactionId());
```
### Listing Indexes
```java
import org.lance.namespace.model.IndexContent;
import org.lance.namespace.model.ListTableIndicesRequest;
import org.lance.namespace.model.ListTableIndicesResponse;
ListTableIndicesRequest request = new ListTableIndicesRequest();
request.setId(Arrays.asList("my_namespace", "my_table"));
ListTableIndicesResponse response = namespaceClient.listTableIndices(request);
for (IndexContent index : response.getIndexes()) {
System.out.println(index.getIndexName() + ": " + index.getStatus());
}
```
!!! note
The current Java namespace API exposes index type, index name, distance type, and full text search tokenizer options.
IVF training parameters such as `num_partitions` are not exposed by `CreateTableIndexRequest` yet.
To make those configurable from Java, the namespace API must add those fields first.
## Reading Query Results
Query results are returned as bytes in Apache Arrow IPC file format. Put the byte-channel
adapter behind a small helper so query code can work with `ArrowFileReader` directly:
Query results are returned in Apache Arrow IPC file format. Here's how to read them:
```java
import org.apache.arrow.vector.ipc.ArrowFileReader;
@@ -571,50 +441,45 @@ import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SeekableByteChannel;
final class ArrowIpc {
static ArrowFileReader openFileReader(byte[] data, BufferAllocator allocator) throws IOException {
return new ArrowFileReader(new ByteArraySeekableByteChannel(data), allocator);
// Helper class to read Arrow data from byte array
class ByteArraySeekableByteChannel implements SeekableByteChannel {
private final byte[] data;
private long position = 0;
private boolean isOpen = true;
public ByteArraySeekableByteChannel(byte[] data) {
this.data = data;
}
private static final class ByteArraySeekableByteChannel implements SeekableByteChannel {
private final byte[] data;
private long position = 0;
private boolean isOpen = true;
private ByteArraySeekableByteChannel(byte[] data) {
this.data = data;
}
@Override
public int read(ByteBuffer dst) {
int remaining = dst.remaining();
int available = (int) (data.length - position);
if (available <= 0) return -1;
int toRead = Math.min(remaining, available);
dst.put(data, (int) position, toRead);
position += toRead;
return toRead;
}
@Override public long position() { return position; }
@Override public SeekableByteChannel position(long newPosition) { position = newPosition; return this; }
@Override public long size() { return data.length; }
@Override public boolean isOpen() { return isOpen; }
@Override public void close() { isOpen = false; }
@Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
@Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
@Override
public int read(ByteBuffer dst) {
int remaining = dst.remaining();
int available = (int) (data.length - position);
if (available <= 0) return -1;
int toRead = Math.min(remaining, available);
dst.put(data, (int) position, toRead);
position += toRead;
return toRead;
}
@Override public long position() { return position; }
@Override public SeekableByteChannel position(long newPosition) { position = newPosition; return this; }
@Override public long size() { return data.length; }
@Override public boolean isOpen() { return isOpen; }
@Override public void close() { isOpen = false; }
@Override public int write(ByteBuffer src) { throw new UnsupportedOperationException(); }
@Override public SeekableByteChannel truncate(long size) { throw new UnsupportedOperationException(); }
}
// Read query results
byte[] queryResult = namespaceClient.queryTable(query);
try (BufferAllocator allocator = new RootAllocator();
ArrowFileReader reader = ArrowIpc.openFileReader(queryResult, allocator)) {
ArrowFileReader reader = new ArrowFileReader(
new ByteArraySeekableByteChannel(queryResult), allocator)) {
for (int i = 0; i < reader.getRecordBlocks().size(); i++) {
reader.loadRecordBatch(reader.getRecordBlocks().get(i));
-13
View File
@@ -43,19 +43,6 @@ The following tokenizers are available:
***
### blockSize?
```ts
optional blockSize: 128 | 256;
```
Number of documents per compressed posting block.
The default is 128. Supported values are 128 and 256. A value of 256 uses
the experimental FTS V3 format and may introduce breaking changes.
***
### language?
```ts
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.32.0-beta.3</version>
<version>0.32.0-beta.2</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.32.0-beta.3</version>
<version>0.32.0-beta.2</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>10.0.0-beta.5</lance-core.version>
<lance-core.version>9.0.0-rc.2</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.32.0-beta.3"
version = "0.32.0-beta.2"
publish = false
license.workspace = true
description.workspace = true
-49
View File
@@ -15,7 +15,6 @@ import {
OAuthHeaderProvider,
StaticHeaderProvider,
} from "../lancedb/header";
import { Index } from "../lancedb/indices";
// Test-only header providers
class CustomProvider extends HeaderProvider {
@@ -226,54 +225,6 @@ describe("remote connection", () => {
);
});
it("sends the FTS posting block size to remote tables", async () => {
let createIndexBody: Record<string, unknown> | undefined;
await withMockDatabase(
(req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "t",
version: 1,
schema: {
fields: [
{ name: "text", type: { type: "string" }, nullable: false },
],
},
}),
);
return;
}
if (path.endsWith("/create_index/")) {
let raw = "";
req.on("data", (chunk) => {
raw += chunk;
});
req.on("end", () => {
createIndexBody = JSON.parse(raw);
res.writeHead(200).end();
});
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("t");
await table.createIndex("text", {
config: Index.fts({ blockSize: 256 }),
});
},
);
expect(createIndexBody?.["column"]).toBe("text");
expect(createIndexBody?.["index_type"]).toBe("FTS");
expect(createIndexBody?.["block_size"]).toBe(256);
});
it("diffs and merges remote branches", async () => {
const sampleDiff = {
fromBranch: "exp",
-29
View File
@@ -2527,35 +2527,6 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(results3.length).toBe(1);
});
test("full text search with custom posting block size", async () => {
const db = await connect(tmpDir.name);
const data = [
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
{ text: "goodbye world", vector: [0.4, 0.5, 0.6] },
];
const table = await db.createTable("test", data);
await table.createIndex("text", {
config: Index.fts({ blockSize: 256 }),
});
const index = (await table.listIndices()).find(
(index) => index.indexType === "FTS",
);
expect(index?.indexVersion).toBe(3);
expect(
(index?.indexDetails as Record<string, unknown>)["block_size"],
).toBe(256);
const results = await table.search("hello").toArray();
expect(results[0].text).toBe(data[0].text);
});
test("rejects invalid full text posting block size", () => {
expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
"128 or 256",
);
});
test("full text search without lowercase", async () => {
const db = await connect(tmpDir.name);
const data = [
-9
View File
@@ -572,14 +572,6 @@ export interface FtsOptions {
* whether to only index the prefix of the token for ngram tokenizer
*/
prefixOnly?: boolean;
/**
* Number of documents per compressed posting block.
*
* The default is 128. Supported values are 128 and 256. A value of 256 uses
* the experimental FTS V3 format and may introduce breaking changes.
*/
blockSize?: 128 | 256;
}
export class Index {
@@ -759,7 +751,6 @@ export class Index {
options?.ngramMinLength,
options?.ngramMaxLength,
options?.prefixOnly,
options?.blockSize,
),
);
}
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"cpu": [
"x64",
"arm64"
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.32.0-beta.3",
"version": "0.32.0-beta.2",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
+3 -9
View File
@@ -226,8 +226,7 @@ impl Index {
ngram_min_length: Option<u32>,
ngram_max_length: Option<u32>,
prefix_only: Option<bool>,
block_size: Option<u32>,
) -> napi::Result<Self> {
) -> Self {
let mut opts = FtsIndexBuilder::default();
if let Some(with_position) = with_position {
opts = opts.with_position(with_position);
@@ -262,15 +261,10 @@ impl Index {
if let Some(prefix_only) = prefix_only {
opts = opts.ngram_prefix_only(prefix_only);
}
if let Some(block_size) = block_size {
opts = opts
.block_size(block_size as usize)
.map_err(|err| napi::Error::from_reason(err.to_string()))?;
}
Ok(Self {
Self {
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
})
}
}
#[napi(factory)]
@@ -2,7 +2,7 @@
Manage branches on a LanceDB table: list what exists, create new ones, delete stale ones, and direct read/write operations at a specific branch without touching main. Use for branch lifecycle tasks, experimental/isolated table versions, targeting an operation at a non-main branch, or confirming a mutation did not affect main.
Works on local/OSS and remote Enterprise/Cloud tables, except merging a branch into main, which is Enterprise-only.
Works on local/OSS and remote Enterprise/Cloud tables.
## The branch model (important)
@@ -101,69 +101,6 @@ assert b"lancedb:description" not in (table.schema.field("category").metadata or
Two handles on the same branch see each other's writes (e.g. `table.branches.create("exp")` and `db.open_table(name, branch="exp")`); main stays isolated.
## Merging a branch into main (Enterprise only)
Merge is available through the SDKs (`table.branches.merge(...)`) on **Enterprise tables only** — it is not supported on Cloud or local/OSS tables, which raise `NotSupported`.
`merge` takes the branch to merge **from** and a `dry_run` flag. Both the SDK method and the underlying REST endpoint **actually merge by default** (`dry_run=False`); pass `dry_run=True` to only preview. A rejected merge is **not an exception** — it returns a result with `status="rejected"` rather than raising, so inspect the return value. Use `branches.diff(from_branch)` to inspect a branch's pending diff without attempting a merge.
```python
exp = "experiment-reindex"
# preview only — returns status="ready" if it would merge cleanly
preview = table.branches.merge(exp, dry_run=True)
# actually merge (default)
result = table.branches.merge(exp)
if result["status"] == "merged":
print("landed at", result["mainVersionAfter"])
elif result["status"] == "rejected":
print(result["diff"]["mergeBlockers"]) # why it was refused
# inspect a branch's pending diff without merging
diff = table.branches.diff(exp)
```
Async: `await table.branches.merge(exp)`, `await table.branches.diff(exp)`.
```typescript
const branches = await table.branches();
const exp = "experiment-reindex";
// preview only (second arg is dryRun)
const preview = await branches.merge(exp, true);
// actually merge (default)
const result = await branches.merge(exp);
if (result.status === "merged") {
console.log("landed at", result.mainVersionAfter);
} else if (result.status === "rejected") {
console.log(result.diff.mergeBlockers);
}
const diff = await branches.diff(exp);
```
The result is the wire JSON, containing `status` (`ready` on a passing dry run, `merged` on success, `rejected` when refused — also `notImplemented`/`unknown`), the branch `diff` (including `mergeBlockers` explaining any rejection), a `preview` of the columns that would be promoted, and — after a real merge — `mainVersionAfter`.
### Merge preconditions
Merge only **promotes newly added columns** onto main; it does not replay arbitrary commits. Practically, a branch is mergeable only if it has **exactly one commit since it was created, and that commit added a column**. The merge is rejected (`status: "rejected"`, with `mergeBlockers` set) if:
- the branch was forked from another branch rather than directly from main
- main has advanced since the branch was forked
- the branch's rows changed since the fork (row counts must match main exactly)
- the branch removed columns or changed a column's type/nullability
- the branch added no columns (index-only changes are not merged)
### Adding a column in a single commit
Because the branch must contain just one column-adding commit, add the column with its values in one operation rather than add-then-backfill:
1. **SQL transformation**`add_columns` with a SQL expression computed from existing columns, so the column lands populated in one commit.
2. **Precompute the values** — compute the column's values externally, then add the fully-populated column in a single operation (e.g. via `merge_insert`/`add_columns` with the data ready).
3. **Lance-format-level data evolution (pylance)** — use Lance's data evolution with backfill, documented at <https://lance.org/guide/data_evolution/#with-data-backfill>.
## Quick reference
| Goal | Python | TypeScript |
@@ -176,7 +113,5 @@ Because the branch must contain just one column-adding commit, add the column wi
| Delete branch | `table.branches.delete(name)` | `await branches.delete(name)` |
| Which branch is this handle on? | `table.current_branch()` (`None` = main) | `table.currentBranch()` (`null` = main) |
| Target main | use the original (non-branch) handle | use the original (non-branch) handle |
| Merge branch into main (Enterprise only) | `table.branches.merge(from_branch, dry_run=False)` | `await branches.merge(fromBranch, dryRun)` |
| Preview a branch's pending diff (Enterprise only) | `table.branches.diff(from_branch)` | `await branches.diff(fromBranch)` |
Branch names must be non-empty; empty names raise a validation error.
@@ -4,19 +4,12 @@ Quick method reference for Python LanceDB code. Cross-check source for non-trivi
## Connect
If you're connecting to a remote database, use this:
```python
import lancedb
db = lancedb.connect("db://my-db", api_key=api_key, host_override=host_override) # remote
db = lancedb.connect("./camelot-db") # local/OSS
db = lancedb.connect("db://my-db", api_key=api_key, region=region) # remote
```
(values may be found in LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, either in env vars or a .env file)
If you're connecting to a local table using OSS LanceDB, use this:
```python
db = lancedb.connect("./camelot-db") # local/OSS
```
If you're not sure which, or if you can't find the api_key or host_override params, ask the user.
**Place the local database directory next to the script/entrypoint that opens it** (i.e. resolve the path relative to the script, `Path(__file__).parent / "camelot-db"`), not buried under a shared `data/` folder. The Lance dataset is the database, not a data file — keeping it beside its code makes ownership obvious and paths stable regardless of the working directory the script is launched from.
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.35.0-beta.3"
current_version = "0.35.0-beta.2"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.35.0-beta.3"
version = "0.35.0-beta.2"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
-11
View File
@@ -115,12 +115,6 @@ class FTS:
For example, it works with `title`, `description`, `content`, etc.
Examples
--------
Create an index configuration that uses 256-document posting blocks:
>>> config = FTS(block_size=256)
Attributes
----------
with_position : bool, default False
@@ -154,10 +148,6 @@ class FTS:
ascii_folding : bool, default True
Whether to fold ASCII characters. This converts accented characters to
their ASCII equivalent. For example, "café" would be converted to "cafe".
block_size : int, default 128
The number of documents per compressed posting block. Supported values
are 128 and 256. A value of 256 uses the experimental FTS V3 format
and may introduce breaking changes.
Notes
-----
@@ -178,7 +168,6 @@ class FTS:
ngram_min_length: int = 3
ngram_max_length: int = 3
prefix_only: bool = False
block_size: int = 128
@dataclass
+4 -9
View File
@@ -885,7 +885,7 @@ class Permutation:
This method refines the current selection, potentially removing columns. It
will not add back columns that were previously removed.
If any of the columns do not exist then an error will be raised.
If any of the columns do not exist then an error will be raised
This does not introduce a post-processing step. It simply reduces the amount
of data we read.
@@ -898,14 +898,9 @@ class Permutation:
for name in columns:
value = self.selection.get(name, None)
if value is None:
if name == "_rowid":
# _rowid is a system column not in the default schema
# but can be explicitly selected
value = "_rowid"
else:
raise ValueError(
f"Cannot select column `{name}` because it does not exist"
)
raise ValueError(
f"Cannot select column `{name}` because it does not exist"
)
new_selection[name] = value
return self._with_selection(new_selection)
-2
View File
@@ -344,7 +344,6 @@ class RemoteTable(Table):
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -365,7 +364,6 @@ class RemoteTable(Table):
ngram_min_length=ngram_min_length,
ngram_max_length=ngram_max_length,
prefix_only=prefix_only,
block_size=block_size,
)
LOOP.run(
self._table.create_index(
@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
column : str, default "text"
The name of the column to use as input to the cross encoder model.
return_score : str, default "relevance"
options are "relevance" or "all".
options are "relevance" or "all". Only "relevance" is supported for now.
**kwargs
Additional keyword arguments to pass to the model. For example, 'device'.
See AnswerDotAI/rerankers for more information.
@@ -77,13 +77,12 @@ class AnswerdotaiRerankers(Reranker):
vector_results: pa.Table,
fts_results: pa.Table,
):
if self.score == "all":
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
else:
combined_results = self.merge_results(vector_results, fts_results)
combined_results = self.merge_results(vector_results, fts_results)
combined_results = self._rerank(combined_results, query)
if self.score == "relevance":
combined_results = self._keep_relevance_score(combined_results)
elif self.score == "all":
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
combined_results = combined_results.sort_by(
[("_relevance_score", "descending")]
)
+1 -1
View File
@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
column : str, default "text"
The name of the column to use as input to the cross encoder model.
return_score : str, default "relevance"
options are "relevance" or "all".
options are "relevance" or "all". Only "relevance" is supported for now.
**kwargs
Additional keyword arguments to pass to the model, for example, 'device'.
See AnswerDotAI/rerankers for more information.
+3 -7
View File
@@ -1106,7 +1106,6 @@ class Table(ABC):
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
wait_timeout: Optional[timedelta] = None,
name: Optional[str] = None,
):
@@ -1178,10 +1177,6 @@ class Table(ABC):
The maximum length of an n-gram.
prefix_only: bool, default False
Whether to only index the prefix of the token for ngram tokenizer.
block_size: int, default 128
The number of documents per compressed posting block. Must be 128
or 256. A value of 256 uses the experimental FTS V3 format and
may introduce breaking changes.
wait_timeout: timedelta, optional
The timeout to wait if indexing is asynchronous.
name: str, optional
@@ -3031,7 +3026,6 @@ class LanceTable(Table):
ngram_min_length: int = 3,
ngram_max_length: int = 3,
prefix_only: bool = False,
block_size: int = 128,
name: Optional[str] = None,
):
"""Create a full-text search index on a column.
@@ -3081,7 +3075,9 @@ class LanceTable(Table):
else:
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
config = FTS(block_size=block_size, **tokenizer_configs)
config = FTS(
**tokenizer_configs,
)
try:
LOOP.run(
-17
View File
@@ -226,23 +226,6 @@ def test_create_inverted_index(table, with_position):
assert any(i.name == "custom_fts_index" for i in fts_indices)
@pytest.mark.parametrize("block_size", [128, 256])
def test_create_inverted_index_block_size(table, block_size):
table.create_index("text", config=FTS(block_size=block_size))
index = next(index for index in table.list_indices() if index.index_type == "FTS")
assert index.index_details["block_size"] == block_size
assert index.index_version == (2 if block_size == 128 else 3)
results = table.search("puppy").limit(5).to_list()
assert len(results) == 5
def test_create_inverted_index_rejects_invalid_block_size(table):
with pytest.raises(ValueError, match="128 or 256"):
table.create_index("text", config=FTS(block_size=129))
def test_search_fts(table):
table.create_fts_index("text")
results = table.search("puppy").select(["id", "text"]).limit(5).to_list()
-58
View File
@@ -1136,61 +1136,3 @@ def test_take_offsets_empty_permutation(some_permutation: Permutation):
result = some_permutation.take_offsets([])
assert result == []
def test_select_rowid(some_permutation: Permutation):
"""Test that _rowid can be selected alongside regular columns."""
perm_with_rowid = some_permutation.select_columns(["_rowid", "id"])
assert "_rowid" in perm_with_rowid.column_names
batches = list(perm_with_rowid.iter(100, skip_last_batch=False))
for batch in batches:
assert "_rowid" in batch[0]
def test_select_rowid_only(some_permutation: Permutation):
"""Test that _rowid can be selected as the sole column."""
perm_rowid_only = some_permutation.select_columns(["_rowid"])
assert perm_rowid_only.column_names == ["_rowid"]
batches = list(perm_rowid_only.iter(100, skip_last_batch=False))
assert len(batches) > 0
for batch in batches:
assert list(batch[0].keys()) == ["_rowid"]
def test_select_rowid_not_in_default(some_permutation: Permutation):
"""Test that _rowid is NOT in the default column_names or schema."""
assert "_rowid" not in some_permutation.column_names
assert "_rowid" not in some_permutation.schema.names
def test_select_rowid_identity_permutation(mem_db):
"""Test that _rowid works with an identity permutation."""
tbl = mem_db.create_table(
"test_rowid_identity", pa.table({"id": range(10), "value": range(10)})
)
perm = Permutation.identity(tbl)
perm_with_rowid = perm.select_columns(["_rowid", "id"])
batches = list(perm_with_rowid.iter(10, skip_last_batch=False))
assert len(batches) == 1
assert "_rowid" in batches[0][0]
def test_rename_rowid(some_permutation: Permutation):
"""Test that _rowid can be selected and then renamed."""
perm_with_rowid = some_permutation.select_columns(["_rowid", "id"])
renamed = perm_with_rowid.rename_column("_rowid", "my_row_id")
assert "my_row_id" in renamed.column_names
assert "_rowid" not in renamed.column_names
batches = list(renamed.iter(100, skip_last_batch=False))
for batch in batches:
assert "my_row_id" in batch[0]
assert "_rowid" not in batch[0]
def test_remove_rowid_after_select(some_permutation: Permutation):
"""Test that _rowid can be selected and then removed."""
perm_with_rowid = some_permutation.select_columns(["_rowid", "id"])
assert "_rowid" in perm_with_rowid.column_names
perm_without_rowid = perm_with_rowid.remove_columns(["_rowid"])
assert "_rowid" not in perm_without_rowid.column_names
assert perm_without_rowid.column_names == ["id"]
+2 -7
View File
@@ -768,10 +768,7 @@ def test_table_create_indices():
# Test create_fts_index with custom name (legacy method)
with pytest.warns(DeprecationWarning, match="create_fts_index"):
table.create_fts_index(
"text",
wait_timeout=timedelta(seconds=2),
block_size=256,
name="custom_fts_idx",
"text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
)
# Test create_index with custom name (legacy form: vector_column_name kwarg)
@@ -794,7 +791,6 @@ def test_table_create_indices():
fts_req = received_requests[1]
assert "name" in fts_req
assert fts_req["name"] == "custom_fts_idx"
assert fts_req["block_size"] == 256
# Check vector index request has custom name
vector_req = received_requests[2]
@@ -880,7 +876,7 @@ def test_remote_create_index_new_api():
_warnings.simplefilter("error", DeprecationWarning)
table.create_index("vector", config=IvfPq(distance_type="l2"))
table.create_index("category", config=BTree())
table.create_index("text", config=FTS(block_size=256))
table.create_index("text", config=FTS())
# IvfRq via new API
table.create_index("vector", config=IvfRq(distance_type="l2"))
@@ -900,7 +896,6 @@ def test_remote_create_index_new_api():
"vector",
"vector",
]
assert received_requests[2]["block_size"] == 256
def test_table_wait_for_index_timeout():
-15
View File
@@ -644,21 +644,6 @@ def test_cross_encoder_reranker_return_all(tmp_path):
assert "_distance" in result.column_names
def test_answerdotai_reranker_return_all(tmp_path):
pytest.importorskip("rerankers")
reranker = AnswerdotaiRerankers(return_score="all")
table, schema = get_test_table(tmp_path)
query = "single player experience"
result = (
table.search(query, query_type="hybrid", vector_column_name="vector")
.rerank(reranker=reranker)
.to_arrow()
)
assert "_relevance_score" in result.column_names
assert "_score" in result.column_names
assert "_distance" in result.column_names
# ---------------------------------------------------------------------------
# Regression tests for LinearCombinationReranker scoring bugs (issue #3154)
# ---------------------------------------------------------------------------
-4
View File
@@ -60,9 +60,6 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
.ngram_min_length(params.ngram_min_length)
.ngram_max_length(params.ngram_max_length)
.ngram_prefix_only(params.prefix_only);
let inner_opts = inner_opts
.block_size(params.block_size)
.map_err(|err| PyValueError::new_err(err.to_string()))?;
Ok(LanceDbIndex::FTS(inner_opts))
}
"IvfFlat" => {
@@ -210,7 +207,6 @@ struct FtsParams {
ngram_min_length: u32,
ngram_max_length: u32,
prefix_only: bool,
block_size: usize,
}
#[derive(FromPyObject)]
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.32.0-beta.3"
version = "0.32.0-beta.2"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
+2 -6
View File
@@ -273,11 +273,7 @@ pub(crate) async fn take_blobs_aligned(
if *is_null {
builder.append_null();
} else {
if let Some(data) = &payloads[payload_idx].data {
builder.append_value(data);
} else {
builder.append_null();
}
builder.append_value(payloads[payload_idx].data.as_ref());
payload_idx += 1;
}
}
@@ -319,7 +315,7 @@ pub(crate) async fn take_blob_files_aligned(
if *is_null {
None
} else {
handles.next().flatten()
Some(handles.next().unwrap())
}
})
.collect())
+1 -20
View File
@@ -54,26 +54,7 @@ pub enum Index {
/// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
Fm(FmIndexBuilder),
/// Full text search index using BM25.
///
/// The posting block size defaults to 128. Supported values are 128 and 256;
/// a value of 256 uses the experimental FTS V3 format and may introduce
/// breaking changes.
///
/// ```
/// use lancedb::index::{Index, scalar::FtsIndexBuilder};
///
/// # async fn create_fts_index(
/// # table: &lancedb::Table,
/// # ) -> Result<(), Box<dyn std::error::Error>> {
/// let params = FtsIndexBuilder::default().block_size(256)?;
/// table
/// .create_index(&["text"], Index::FTS(params))
/// .execute()
/// .await?;
/// # Ok(())
/// # }
/// ```
/// Full text search index using bm25.
FTS(FtsIndexBuilder),
/// IVF index
-9
View File
@@ -4496,15 +4496,6 @@ mod tests {
serde_json::to_value(InvertedIndexParams::default()).unwrap(),
Index::FTS(Default::default()),
),
(
"FTS",
{
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
body["block_size"] = 256.into();
body
},
Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
),
];
for (index_type, expected_body, index) in cases {
+2 -11
View File
@@ -382,9 +382,7 @@ mod tests {
use crate::connect;
use crate::connection::ConnectBuilder;
use crate::index::Index;
use crate::index::scalar::{
BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
};
use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder};
use crate::index::vector::{
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
};
@@ -1364,10 +1362,7 @@ mod tests {
.unwrap();
table
.create_index(
&["text"],
Index::FTS(FtsIndexBuilder::default().block_size(256).unwrap()),
)
.create_index(&["text"], Index::FTS(Default::default()))
.execute()
.await
.unwrap();
@@ -1377,10 +1372,6 @@ mod tests {
assert_eq!(index.index_type, crate::index::IndexType::FTS);
assert_eq!(index.columns, vec!["text".to_string()]);
assert_eq!(index.name, "text_idx");
assert_eq!(index.index_version, Some(3));
let index_params: FtsIndexBuilder =
serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
assert_eq!(index_params.posting_block_size(), 256);
let num_rows = 120;
let stats = table.index_stats("text_idx").await.unwrap().unwrap();
+11
View File
@@ -774,6 +774,9 @@ fn shard_writer_config_from_defaults(defaults: &HashMap<String, String>) -> Shar
if let Some(v) = bool_of("durable_write") {
config = config.with_durable_write(v);
}
if let Some(v) = bool_of("sync_indexed_write") {
config = config.with_sync_indexed_write(v);
}
if let Some(v) = usize_of("max_wal_buffer_size") {
config = config.with_max_wal_buffer_size(v);
}
@@ -795,6 +798,12 @@ fn shard_writer_config_from_defaults(defaults: &HashMap<String, String>) -> Shar
if let Some(v) = millis_of("backpressure_log_interval_ms") {
config = config.with_backpressure_log_interval(v);
}
if let Some(v) = usize_of("async_index_buffer_rows") {
config = config.with_async_index_buffer_rows(v);
}
if let Some(v) = millis_of("async_index_interval_ms") {
config = config.with_async_index_interval(v);
}
if let Some(v) = bool_of("enable_memtable") {
config = config.with_enable_memtable(v);
}
@@ -999,11 +1008,13 @@ mod tests {
let defaults = HashMap::from([
("durable_write".to_string(), "false".to_string()),
("max_memtable_rows".to_string(), "4096".to_string()),
("async_index_interval_ms".to_string(), "250".to_string()),
("unknown_key".to_string(), "ignored".to_string()),
]);
let config = shard_writer_config_from_defaults(&defaults);
assert!(!config.durable_write);
assert_eq!(config.max_memtable_rows, 4096);
assert_eq!(config.async_index_interval, Duration::from_millis(250));
assert_eq!(config.shard_spec_id, SHARDING_SPEC_ID);
}