From a5079b0fc6c47100f8f5030b8dd99e40c9d576de Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 22 Oct 2026 19:41:40 +0800 Subject: [PATCH] chore: update lance dependency to v15.0.0-beta.1 (#4628) --- .github/workflows/npm-publish.yml | 32 +++---- Cargo.lock | 152 +++++++++++++++++++++--------- Cargo.toml | 28 +++--- docs/web/indexing/fts-index.mdx | 8 +- java/pom.xml | 2 +- rust/lancedb/src/index/scalar.rs | 11 +++ 6 files changed, 154 insertions(+), 79 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index c3724de8d..a8dd3993e 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -71,9 +71,9 @@ jobs: # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian pre_build: |- - set -e && - apt-get update && - apt-get install -y protobuf-compiler pkg-config && + set -e + apt-get update + apt-get install -y protobuf-compiler pkg-config # The base image (manylinux2014-cross) sets TARGET_CC to the old # GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC, # so it picks up GCC even though the napi-rs image sets CC=clang. @@ -84,10 +84,10 @@ jobs: host: ubuntu-2404-8x-x64 features: fp16kernels pre_build: |- - set -e && - sudo apt-get update && - sudo apt-get install -y protobuf-compiler pkg-config && - rustup target add x86_64-unknown-linux-musl && + set -e + sudo apt-get update + sudo apt-get install -y protobuf-compiler pkg-config + rustup target add x86_64-unknown-linux-musl export EXTRA_ARGS="-x" - target: aarch64-unknown-linux-gnu host: ubuntu-2404-8x-x64 @@ -103,13 +103,13 @@ jobs: # linker process 7.0 -> 4.0 GiB (lancedb/sophon#7313). linker: /tmp/aarch64-lld-clang pre_build: |- - set -e && - apt-get update && - apt-get install -y protobuf-compiler pkg-config && - export TARGET_CC=clang TARGET_CXX=clang++ && + set -e + apt-get update + apt-get install -y protobuf-compiler pkg-config + export TARGET_CC=clang TARGET_CXX=clang++ # The manylinux2014 sysroot has glibc 2.17 headers which lack # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys. - export CFLAGS="$CFLAGS -DAT_HWCAP2=26" && + export CFLAGS="$CFLAGS -DAT_HWCAP2=26" rustup target add aarch64-unknown-linux-gnu # Not `&&`-chained: in dash, errexit does not fire for a # non-final command in an `&&` list, so failures were ignored. @@ -136,10 +136,10 @@ jobs: lto: thin codegen_units: 16 pre_build: |- - set -e && - sudo apt-get update && - sudo apt-get install -y protobuf-compiler && - rustup target add aarch64-unknown-linux-musl && + set -e + sudo apt-get update + sudo apt-get install -y protobuf-compiler + rustup target add aarch64-unknown-linux-musl export EXTRA_ARGS="-x" name: build - ${{ matrix.settings.target }} runs-on: ${{ matrix.settings.host }} diff --git a/Cargo.lock b/Cargo.lock index 3b4086203..c72df37e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arc-swap", "arrow", @@ -5135,8 +5135,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-buffer", @@ -5158,7 +5158,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-buffer", @@ -5172,7 +5172,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-schema", @@ -5181,8 +5181,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrayref", "crunchy", @@ -5192,8 +5192,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-buffer", @@ -5230,8 +5230,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow", "arrow-array", @@ -5262,8 +5262,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow", "arrow-array", @@ -5280,8 +5280,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "proc-macro2", "quote", @@ -5290,8 +5290,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-arith", "arrow-array", @@ -5324,8 +5324,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-arith", "arrow-array", @@ -5357,8 +5357,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "datafusion", "geo-traits", @@ -5372,8 +5372,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arc-swap", "arrow", @@ -5387,6 +5387,7 @@ dependencies = [ "async-recursion", "async-trait", "bitvec", + "blake3", "bytes", "crossbeam-queue", "datafusion", @@ -5421,6 +5422,7 @@ dependencies = [ "lance-tokenizer", "libsais-rs", "log", + "nalgebra", "ndarray", "num-traits", "object_store 0.14.2", @@ -5444,8 +5446,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-schema", @@ -5468,8 +5470,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow", "arrow-array", @@ -5512,8 +5514,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "byteorder", "ethnum", @@ -5530,8 +5532,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-schema", @@ -5545,8 +5547,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow", "async-trait", @@ -5560,8 +5562,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arc-swap", "arrow", @@ -5615,8 +5617,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-buffer", @@ -5630,8 +5632,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow", "arrow-array", @@ -5671,8 +5673,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "arrow-array", "arrow-schema", @@ -5685,8 +5687,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.23" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.23#ea81833500fe251568854dfd889524bc4ebcbaea" +version = "15.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.1#46a7cfe1fba5efa1f90deea4c7b14d414b991409" dependencies = [ "frostem", "icu_segmenter", @@ -6411,6 +6413,21 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "nalgebra" +version = "0.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d43ddcacf343185dfd6de2ee786d9e8b1c2301622afab66b6c73baf9882abfd" +dependencies = [ + "approx", + "matrixmultiply", + "num-complex", + "num-rational", + "num-traits", + "simba", + "typenum", +] + [[package]] name = "napi" version = "3.12.5" @@ -6650,6 +6667,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -9268,6 +9296,15 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "safetensors" version = "0.7.0" @@ -9676,6 +9713,19 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simba" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste", + "wide", +] + [[package]] name = "simd-adler32" version = "0.3.9" @@ -11182,6 +11232,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 49f7ea6a6..3ac246149 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.23", default-features = false, "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.23", default-features = false, "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.23", default-features = false, "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.23", "tag" = "v14.0.0-beta.23", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=15.0.0-beta.1", default-features = false, "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=15.0.0-beta.1", "tag" = "v15.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/web/indexing/fts-index.mdx b/docs/web/indexing/fts-index.mdx index 832b67315..4660e8b01 100644 --- a/docs/web/indexing/fts-index.mdx +++ b/docs/web/indexing/fts-index.mdx @@ -99,7 +99,7 @@ await async_table.create_index("payload.text", config=FTS(with_position=True)) | `stem` | bool | `True` | Apply stemming (`running` → `run`) | | `remove_stop_words` | bool | `True` | Drop common stop words | | `ascii_folding` | bool | `True` | Normalize accented characters | -| `custom_stop_words` | list[str] | `None` | Extra stop words to drop in addition to the language defaults. Requires `remove_stop_words=True`. | +| `custom_stop_words` | list[str] | `None` | Explicit stop-word list that replaces the built-in list. Requires `remove_stop_words=True`. | | `ngram_min_length` | int | `3` | Minimum n-gram length. Applies only when `base_tokenizer="ngram"`. | | `ngram_max_length` | int | `3` | Maximum n-gram length. Applies only when `base_tokenizer="ngram"`. | | `prefix_only` | bool | `False` | Index only prefix n-grams rather than all substrings. Applies only when `base_tokenizer="ngram"`. | @@ -123,7 +123,11 @@ await async_table.create_index("payload.text", config=FTS(with_position=True)) Model-backed tokenizers such as `jieba/default`, `lindera/ipadic`, and `lindera/ko-dic` require tokenizer model files in Lance's language model home. Lance looks under the default platform data directory for `lance/language_models`, or you can set `LANCE_LANGUAGE_MODEL_HOME` to point to another model root. For example, `jieba/default` is resolved under `/jieba/default/...`. For a table in LanceDB Enterprise, the files must be present where the deployment builds and queries the index, so ask its administrator whether they are installed. -`language` is used by token filters, not by the base tokenizer. Stemming supports Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. Built-in stop-word removal supports Danish, Dutch, English, Finnish, French, German, Hungarian, Italian, Norwegian, Portuguese, Russian, Spanish, and Swedish. For other stemming languages, set `remove_stop_words=False` or pass `custom_stop_words`. +`language` is used by token filters, not by the base tokenizer. Stemming and language-specific stop-word removal support Arabic, Danish, Dutch, English, Finnish, French, German, Greek, Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian, Spanish, Swedish, Tamil, and Turkish. The ICU tokenizer uses a combined multilingual stop-word list. An explicit `custom_stop_words` list replaces the built-in list for either tokenizer. + + +Lance `15.0.0-beta.1` revises the built-in stop-word lists, including the multilingual list used by ICU. Existing index metadata records tokenizer settings, but does not freeze these built-in lists. Rebuild affected FTS indexes with the same configuration before serving queries with the updated SDK, so stored terms and query tokens use the same lists. Indexes with stop-word removal disabled or an explicit `custom_stop_words` list are unaffected by this change. + ### Posting block size {#posting-block-size} diff --git a/java/pom.xml b/java/pom.xml index db0ee33d7..5cc7e365e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.23 + 15.0.0-beta.1 2.18.11 false 2.30.0 diff --git a/rust/lancedb/src/index/scalar.rs b/rust/lancedb/src/index/scalar.rs index 5bfc2b4e6..5d2256381 100644 --- a/rust/lancedb/src/index/scalar.rs +++ b/rust/lancedb/src/index/scalar.rs @@ -65,6 +65,17 @@ pub struct FmIndexBuilder {} /// A ZoneMap index stores min/max summaries for ranges of rows and can /// accelerate range predicates by pruning zones that cannot match. /// +/// ZoneMap indexes use per-fragment write seeds by default for strings, binary +/// data, and supported types wider than eight bytes. Creating a seed-enabled +/// index activates a table capability immediately, before any version 1 seed +/// is written. A mutation retaining a legacy seed-enabled index also activates +/// it. Older clients cannot read or write the resulting snapshots, so upgrade +/// all readers and writers before creating such an index or mutating a table +/// that already has one. The capability +/// remains set in subsequent snapshots, including after dropping the index or +/// restoring an older version; dropping the index does not restore access for +/// older clients. Opening a table alone does not activate the capability. +/// /// ``` /// use lancedb::{ /// index::{scalar::ZoneMapIndexBuilder, Index},