mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-12 23:02:28 +00:00
chore: update lance dependency to v14.0.0-beta.11 (#4518)
Updates the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core to [v14.0.0-beta.11](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.11). Adapts the LSM resident-index guards to Lance's query-based index API and adds regression coverage for index presence and type checks. Validation: workspace Clippy with all features and warnings denied, Rust formatting, and targeted LSM query unit tests.
This commit is contained in:
LanceDB Robot
Generated
+46
-46
@@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5133,8 +5133,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5156,7 +5156,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5170,7 +5170,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5179,8 +5179,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -5190,8 +5190,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5228,8 +5228,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5260,8 +5260,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5278,8 +5278,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5288,8 +5288,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5322,8 +5322,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5355,8 +5355,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-traits",
|
||||
@@ -5370,8 +5370,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5442,8 +5442,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5466,8 +5466,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5509,8 +5509,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-jsonb"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"ethnum",
|
||||
@@ -5527,8 +5527,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5542,8 +5542,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5557,8 +5557,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5612,8 +5612,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5627,8 +5627,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5668,8 +5668,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5682,8 +5682,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "14.0.0-beta.10"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f"
|
||||
version = "14.0.0-beta.11"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.94.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>14.0.0-beta.10</lance-core.version>
|
||||
<lance-core.version>14.0.0-beta.11</lance-core.version>
|
||||
<jackson.version>2.18.11</jackson.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
|
||||
@@ -33,6 +33,7 @@ use datafusion_physical_plan::expressions::Column;
|
||||
use datafusion_physical_plan::projection::ProjectionExec;
|
||||
use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
|
||||
use lance::Dataset;
|
||||
use lance::dataset::mem_wal::index::{IndexStore, VectorMemQuery};
|
||||
use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables};
|
||||
use lance::dataset::mem_wal::{
|
||||
DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig,
|
||||
@@ -474,7 +475,7 @@ async fn fts_plan(
|
||||
// omits un-compacted documents. Reject rather than mislead.
|
||||
if !index_maintained(dataset, column, details, "InvertedIndexDetails").await?
|
||||
|| !resident_memtables_carry(&in_memory, |memtable| {
|
||||
memtable.index_store.get_fts_by_column(column).is_some()
|
||||
!memtable.index_store.fts_granularities_on(column).is_empty()
|
||||
})
|
||||
{
|
||||
return Err(Error::NotSupported {
|
||||
@@ -712,7 +713,7 @@ async fn vector_plan(
|
||||
// Reject rather than silently omit rows, mirroring the FTS arm.
|
||||
if !index_maintained(dataset, &column, details, "VectorIndexDetails").await?
|
||||
|| !resident_memtables_carry(&in_memory, |memtable| {
|
||||
memtable.index_store.get_hnsw_by_column(&column).is_some()
|
||||
carries_vector_index(&memtable.index_store, &column)
|
||||
})
|
||||
{
|
||||
return Err(Error::NotSupported {
|
||||
@@ -760,6 +761,22 @@ async fn vector_plan(
|
||||
Ok(scanner.create_plan().await?)
|
||||
}
|
||||
|
||||
/// Probe for a resident vector index without imposing a distance metric, matching
|
||||
/// the presence check used before Lance introduced query-based index lookup.
|
||||
fn carries_vector_index(store: &IndexStore, column: &str) -> bool {
|
||||
let probe = VectorMemQuery {
|
||||
vector: arrow_array::FixedSizeListArray::new_null(
|
||||
Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)),
|
||||
1,
|
||||
1,
|
||||
),
|
||||
k: 1,
|
||||
ef: None,
|
||||
distance_type: None,
|
||||
};
|
||||
store.index_answering(column, &probe).is_some()
|
||||
}
|
||||
|
||||
/// Whether `column` stores binary (uint8) vectors, which the LSM vector planner
|
||||
/// does not support.
|
||||
fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool {
|
||||
@@ -804,6 +821,45 @@ mod tests {
|
||||
use super::*;
|
||||
use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress};
|
||||
|
||||
#[test]
|
||||
fn resident_index_checks_distinguish_columns_and_query_types() {
|
||||
use arrow_schema::Field;
|
||||
use lance::dataset::mem_wal::index::MemIndexSpec;
|
||||
|
||||
let schema = ArrowSchema::new(vec![
|
||||
Field::new("id", DataType::Int32, false),
|
||||
Field::new("text", DataType::Utf8, true),
|
||||
Field::new(
|
||||
"vector",
|
||||
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap();
|
||||
let specs = vec![
|
||||
MemIndexSpec::btree("id_idx", 0, "id"),
|
||||
MemIndexSpec::fts("text_idx", 1, "text"),
|
||||
MemIndexSpec::hnsw(
|
||||
"vector_idx",
|
||||
2,
|
||||
"vector",
|
||||
lance_linalg::distance::DistanceType::Cosine,
|
||||
),
|
||||
];
|
||||
let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap();
|
||||
assert!(carries_vector_index(&store, "vector"));
|
||||
for column in ["id", "text", "missing"] {
|
||||
assert!(!carries_vector_index(&store, column));
|
||||
}
|
||||
assert!(!store.fts_granularities_on("text").is_empty());
|
||||
for column in ["id", "vector", "missing"] {
|
||||
assert!(store.fts_granularities_on(column).is_empty());
|
||||
}
|
||||
let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap();
|
||||
assert!(!carries_vector_index(&empty, "vector"));
|
||||
assert!(empty.fts_granularities_on("text").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exclusion_watermark_gates_on_lagging_index_catchup() {
|
||||
let shard = Uuid::from_u128(1);
|
||||
|
||||
Reference in New Issue
Block a user