diff --git a/Cargo.lock b/Cargo.lock index a3d865be8..3fa997cea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5095,26 +5095,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonb" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" -dependencies = [ - "byteorder", - "ethnum", - "fast-float2", - "itoa", - "jiff", - "nom 8.0.0", - "num-traits", - "ordered-float 5.3.0", - "rand 0.9.5", - "serde", - "serde_json", - "zmij", -] - [[package]] name = "kanaria" version = "0.2.0" @@ -5143,8 +5123,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5217,8 +5197,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5232,7 +5212,7 @@ dependencies = [ "futures", "getrandom 0.2.17", "half", - "jsonb", + "lance-jsonb", "num-traits", "rand 0.9.5", ] @@ -5240,7 +5220,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5254,7 +5234,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5263,8 +5243,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrayref", "crunchy", @@ -5274,8 +5254,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5312,8 +5292,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5330,10 +5310,10 @@ dependencies = [ "datafusion-physical-expr", "futures", "half", - "jsonb", "lance-arrow", "lance-core", "lance-geo", + "lance-jsonb", "log", "pin-project", "prost", @@ -5344,8 +5324,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5362,8 +5342,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "proc-macro2", "quote", @@ -5372,8 +5352,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-arith", "arrow-array", @@ -5406,8 +5386,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-arith", "arrow-array", @@ -5439,8 +5419,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "datafusion", "geo-traits", @@ -5454,8 +5434,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5486,7 +5466,6 @@ dependencies = [ "hex", "itertools 0.14.0", "jieba-rs", - "jsonb", "lance-arrow", "lance-arrow-stats", "lance-bitpacking", @@ -5497,6 +5476,7 @@ dependencies = [ "lance-geo", "lance-index-core", "lance-io", + "lance-jsonb", "lance-linalg", "lance-select", "lance-table", @@ -5526,8 +5506,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5550,8 +5530,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5591,10 +5571,28 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-jsonb" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +dependencies = [ + "byteorder", + "ethnum", + "fast-float2", + "itoa", + "jiff", + "nom 8.0.0", + "num-traits", + "ordered-float 5.3.0", + "serde", + "serde_json", + "zmij", +] + [[package]] name = "lance-linalg" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5608,8 +5606,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "async-trait", @@ -5623,8 +5621,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arc-swap", "arrow", @@ -5678,8 +5676,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-buffer", @@ -5693,8 +5691,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow", "arrow-array", @@ -5734,8 +5732,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "arrow-array", "arrow-schema", @@ -5748,8 +5746,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.7" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.7#0fd26effc79c62c244a5baf92cdda06e301baf21" +version = "14.0.0-beta.10" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index ad7c00477..6c05f297a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.7", default-features = false, "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.7", "tag" = "v14.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index d01e30465..b7126dfe3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.7 + 14.0.0-beta.10 false 2.30.0 1.7 diff --git a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs index 000addbe1..543a59589 100644 --- a/rust/lancedb/src/table/datafusion/udtf/vector_query.rs +++ b/rust/lancedb/src/table/datafusion/udtf/vector_query.rs @@ -411,9 +411,7 @@ impl ExecutionPlan for VectorDedupExec { .map_err(|e| datafusion_common::DataFusionError::External(Box::new(e)))?; let mut scanner = dataset.scan(); scanner - .with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter( - removed, - ))) + .with_row_id_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter(removed))) .blob_handling(BlobHandling::AllBinary) .batch_size(MATERIALIZE_BATCH_ROWS) .strict_batch_size(true);