mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-28 17:08:43 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Yang Cen
|
98945e55e5 | ||
 lancedb automation
|
5eedeea120 | ||
 Lance Release
|
a00edef0e6 | ||
|
Lance Release
|
1b2670443e | ||
|
Yang Cen
|
9dc5ec03aa | ||
 Andrew Chen
|
18760f74cd | ||
|
LanceDB Robot
|
c9d07ef6fc |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.33.0-beta.0"
|
||||
current_version = "0.32.0-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+59
-68
@@ -3421,9 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f1155128aba964cf6925c22a667ed763b52c8c653693c4b46f8a79d509a8c1"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4778,9 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23d04bed056e254bc6e31264b031c8492507ca57939586f016924081dcf221a9"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4854,9 +4852,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6e7b87c8183988c40a6bd30a6d8ec588b84e53f702b82f19859dc71ba6c02bc"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4878,8 +4875,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4893,20 +4889,17 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
"half",
|
||||
"lance-arrow-scalar",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a4d84a3f36133c70bf89d306d813a29f8eb8555bba2b84995260867cfafdd5e"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4916,9 +4909,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "238c8a58308e7718d6bd96b53494eb7953fa299778bc4911cc571c3576e9446d"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4941,6 +4933,7 @@ dependencies = [
|
||||
"object_store",
|
||||
"pin-project",
|
||||
"prost",
|
||||
"quick_cache",
|
||||
"rand 0.9.5",
|
||||
"roaring",
|
||||
"serde_json",
|
||||
@@ -4956,9 +4949,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12f0ac4e1cf2f9b1b2fb5ee11bcd04af617bdd6f6cca13726a41c4212220193e"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4988,9 +4980,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd65e7ea88ab28e5d3e91b58a56c02bfd44c47474caae5f8aed1322df1611476"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5007,9 +4998,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf46656b359786f0b73f13936193583972b7af6e53ccb542da87830be18e94b2"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5018,9 +5008,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4725f14fe6cc1b2a5644786b4afa828d0c243064e208fa17378f839243d049c0"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5055,9 +5044,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c161b00eb5813f98f1d6d52e06f1b712f9eebb0a7f535a8dc1e1122ceca7307"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5087,14 +5075,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf7980b0a6287fd46b9308a31e2cf284065d5693bfb43336297f0400172670ad"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
"arrow-ipc",
|
||||
"arrow-ord",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
@@ -5155,9 +5143,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a95f46ac3e4cdd710ca6929226cfb0dd343d5d2370ecb4b40e1c452043f7d27a"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5179,9 +5166,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6deecd351ed6849184cc83000eabb87c8a5bfc519996f92451284498af7b42c"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -5224,9 +5210,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e5a9b99bd1f49bc2fe5afb81323141abc506c818f589de95dbab4f6143d9a88"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5242,9 +5227,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8624fd33a894b5f2eb411cb56588d29138137ce100dee8e335843828e249b860"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5256,9 +5240,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39f0cb9651a65f8eb411d825f34967fff46ba575a76e6ae9becda6ee31c1a974"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5312,9 +5295,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb5c718c141ea4f067203b11a54ccf57f8effbf963f345bc811f4837a660ad57"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5329,9 +5311,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffc125611b4fe46f99f00b5262e71e17bd947a3bc3b81f3a4a604cc9ca290b3f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5370,9 +5351,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2598f24fa7333b49fdfc9945dfe084dafef2fc9ac94b5017ab4fbe4cdcd59318"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5385,9 +5365,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "9.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73060a844ceda2405759b8f0f16a1c586b5b285dbdf7a3784350d048f991cfd3"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
@@ -5400,7 +5379,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.33.0-beta.0"
|
||||
version = "0.32.0-beta.3"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5488,7 +5467,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.33.0-beta.0"
|
||||
version = "0.32.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5513,7 +5492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.36.0-beta.0"
|
||||
version = "0.35.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -7824,6 +7803,18 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=9.0.0", default-features = false }
|
||||
lance-core = "=9.0.0"
|
||||
lance-datagen = "=9.0.0"
|
||||
lance-file = "=9.0.0"
|
||||
lance-io = { "version" = "=9.0.0", default-features = false }
|
||||
lance-index = "=9.0.0"
|
||||
lance-linalg = "=9.0.0"
|
||||
lance-namespace = "=9.0.0"
|
||||
lance-namespace-impls = { "version" = "=9.0.0", default-features = false }
|
||||
lance-table = "=9.0.0"
|
||||
lance-testing = "=9.0.0"
|
||||
lance-datafusion = "=9.0.0"
|
||||
lance-encoding = "=9.0.0"
|
||||
lance-arrow = "=9.0.0"
|
||||
lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
|
||||
+11
-1
@@ -453,6 +453,16 @@ paths:
|
||||
The metric type to use for the index. l2, Cosine, Dot are supported.
|
||||
index_type:
|
||||
type: string
|
||||
custom_stop_words:
|
||||
type: [array, "null"]
|
||||
items:
|
||||
type: string
|
||||
description: |
|
||||
The custom stop-word list for an FTS index. A non-null
|
||||
array replaces the language's built-in stop-word list and is only
|
||||
applied when remove_stop_words is enabled. Null uses the built-in
|
||||
language list, while an empty array explicitly replaces it with no
|
||||
stop words.
|
||||
responses:
|
||||
"200":
|
||||
description: Index successfully created
|
||||
@@ -510,4 +520,4 @@ paths:
|
||||
"401":
|
||||
$ref: "#/components/responses/unauthorized"
|
||||
"404":
|
||||
$ref: "#/components/responses/not_found"
|
||||
$ref: "#/components/responses/not_found"
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.33.0-beta.0</version>
|
||||
<version>0.32.0-beta.3</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -43,6 +43,34 @@ The following tokenizers are available:
|
||||
|
||||
***
|
||||
|
||||
### blockSize?
|
||||
|
||||
```ts
|
||||
optional blockSize: 128 | 256;
|
||||
```
|
||||
|
||||
Number of documents per compressed posting block.
|
||||
|
||||
The default is 128. Supported values are 128 and 256. A value of 256 uses
|
||||
the experimental FTS V3 format and may introduce breaking changes.
|
||||
|
||||
***
|
||||
|
||||
### customStopWords?
|
||||
|
||||
```ts
|
||||
optional customStopWords: string[];
|
||||
```
|
||||
|
||||
Custom stop words that replace the built-in list for `language`.
|
||||
|
||||
This option only affects tokenization when `removeStopWords` is true.
|
||||
|
||||
`undefined` keeps the built-in language list. An empty array explicitly
|
||||
replaces it with no stop words.
|
||||
|
||||
***
|
||||
|
||||
### language?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple".
|
||||
|
||||
***
|
||||
|
||||
### customStopWords?
|
||||
|
||||
```ts
|
||||
optional customStopWords: string[];
|
||||
```
|
||||
|
||||
Custom stop words that replace the built-in list for `language`.
|
||||
|
||||
This option only affects tokenization when `removeStopWords` is true.
|
||||
|
||||
`undefined` keeps the built-in language list. An empty array explicitly
|
||||
replaces it with no stop words.
|
||||
|
||||
***
|
||||
|
||||
### language?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -94,9 +94,33 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
|
||||
|
||||
## Full text search
|
||||
|
||||
Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
|
||||
[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
|
||||
asynchronous API.
|
||||
Pass `custom_stop_words` to [lancedb.index.FTS][]:
|
||||
|
||||
```python
|
||||
from lancedb.index import FTS
|
||||
|
||||
table.create_index(
|
||||
"text",
|
||||
config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]),
|
||||
)
|
||||
```
|
||||
|
||||
The list replaces the built-in stop words and is used only when
|
||||
`remove_stop_words=True`:
|
||||
|
||||
- `custom_stop_words=None` uses the built-in list for `language`.
|
||||
- `custom_stop_words=[]` removes no words.
|
||||
- Values are passed through without trimming, lowercasing, or other rewriting.
|
||||
|
||||
The same option is available on `lancedb.tokenize(...)` and the deprecated
|
||||
[lancedb.table.Table.create_fts_index][] compatibility helper:
|
||||
|
||||
```python
|
||||
import lancedb
|
||||
|
||||
tokens = list(lancedb.tokenize("acme makes searchable data",
|
||||
custom_stop_words=["acme"]))
|
||||
```
|
||||
|
||||
::: lancedb.index.FTS
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.33.0-beta.0</version>
|
||||
<version>0.32.0-beta.3</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.33.0-beta.0</version>
|
||||
<version>0.32.0-beta.3</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>9.0.0</lance-core.version>
|
||||
<lance-core.version>10.0.0-beta.5</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.33.0-beta.0"
|
||||
version = "0.32.0-beta.3"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
OAuthHeaderProvider,
|
||||
StaticHeaderProvider,
|
||||
} from "../lancedb/header";
|
||||
import { Index } from "../lancedb/indices";
|
||||
|
||||
// Test-only header providers
|
||||
class CustomProvider extends HeaderProvider {
|
||||
@@ -225,6 +226,59 @@ describe("remote connection", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("sends FTS options to remote tables", async () => {
|
||||
let createIndexBody: Record<string, unknown> | undefined;
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const path = req.url ?? "";
|
||||
if (path.endsWith("/describe/")) {
|
||||
res.writeHead(200, { "Content-Type": "application/json" }).end(
|
||||
JSON.stringify({
|
||||
name: "t",
|
||||
version: 1,
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "text", type: { type: "string" }, nullable: false },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (path.endsWith("/create_index/")) {
|
||||
let raw = "";
|
||||
req.on("data", (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
createIndexBody = JSON.parse(raw);
|
||||
res.writeHead(200).end();
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("t");
|
||||
await table.createIndex("text", {
|
||||
config: Index.fts({
|
||||
blockSize: 256,
|
||||
removeStopWords: true,
|
||||
customStopWords: ["the"],
|
||||
}),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
expect(createIndexBody?.["column"]).toBe("text");
|
||||
expect(createIndexBody?.["index_type"]).toBe("FTS");
|
||||
expect(createIndexBody?.["block_size"]).toBe(256);
|
||||
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
|
||||
});
|
||||
|
||||
it("diffs and merges remote branches", async () => {
|
||||
const sampleDiff = {
|
||||
fromBranch: "exp",
|
||||
|
||||
@@ -2527,6 +2527,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(results3.length).toBe(1);
|
||||
});
|
||||
|
||||
test("full text search with custom posting block size", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
|
||||
{ text: "goodbye world", vector: [0.4, 0.5, 0.6] },
|
||||
];
|
||||
const table = await db.createTable("test", data);
|
||||
await table.createIndex("text", {
|
||||
config: Index.fts({ blockSize: 256 }),
|
||||
});
|
||||
|
||||
const index = (await table.listIndices()).find(
|
||||
(index) => index.indexType === "FTS",
|
||||
);
|
||||
expect(index?.indexVersion).toBe(3);
|
||||
expect(
|
||||
(index?.indexDetails as Record<string, unknown>)["block_size"],
|
||||
).toBe(256);
|
||||
|
||||
const results = await table.search("hello").toArray();
|
||||
expect(results[0].text).toBe(data[0].text);
|
||||
});
|
||||
|
||||
test("rejects invalid full text posting block size", () => {
|
||||
expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
|
||||
"128 or 256",
|
||||
);
|
||||
});
|
||||
|
||||
test("full text search without lowercase", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
@@ -2732,6 +2761,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
},
|
||||
);
|
||||
|
||||
test("tokenize supports custom stop words", async () => {
|
||||
const tokens = await tokenize("the lance data", {
|
||||
stem: false,
|
||||
removeStopWords: true,
|
||||
customStopWords: ["lance"],
|
||||
});
|
||||
expect(tokens.map((token) => token.text)).toEqual(["the", "data"]);
|
||||
});
|
||||
|
||||
describe("when calling explainPlan", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
|
||||
@@ -29,8 +29,14 @@ test("full text search", async () => {
|
||||
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
|
||||
|
||||
await tbl.createIndex("doc", {
|
||||
config: lancedb.Index.fts(),
|
||||
config: lancedb.Index.fts({
|
||||
stem: false,
|
||||
removeStopWords: true,
|
||||
customStopWords: ["banana"],
|
||||
}),
|
||||
});
|
||||
const tokens = await tbl.tokenize("apple banana", { column: "doc" });
|
||||
expect(tokens.map((token) => token.text)).toEqual(["apple"]);
|
||||
|
||||
// --8<-- [start:full_text_search]
|
||||
const result = await tbl
|
||||
|
||||
@@ -194,6 +194,16 @@ export interface TokenizeOptions {
|
||||
/** Whether to remove stop words. */
|
||||
removeStopWords?: boolean;
|
||||
|
||||
/**
|
||||
* Custom stop words that replace the built-in list for `language`.
|
||||
*
|
||||
* This option only affects tokenization when `removeStopWords` is true.
|
||||
*
|
||||
* `undefined` keeps the built-in language list. An empty array explicitly
|
||||
* replaces it with no stop words.
|
||||
*/
|
||||
customStopWords?: string[];
|
||||
|
||||
/** Whether to fold ASCII characters. */
|
||||
asciiFolding?: boolean;
|
||||
|
||||
@@ -225,6 +235,7 @@ export async function tokenize(
|
||||
options?.lowercase,
|
||||
options?.stem,
|
||||
options?.removeStopWords,
|
||||
options?.customStopWords,
|
||||
options?.asciiFolding,
|
||||
options?.ngramMinLength,
|
||||
options?.ngramMaxLength,
|
||||
|
||||
@@ -553,6 +553,16 @@ export interface FtsOptions {
|
||||
*/
|
||||
removeStopWords?: boolean;
|
||||
|
||||
/**
|
||||
* Custom stop words that replace the built-in list for `language`.
|
||||
*
|
||||
* This option only affects tokenization when `removeStopWords` is true.
|
||||
*
|
||||
* `undefined` keeps the built-in language list. An empty array explicitly
|
||||
* replaces it with no stop words.
|
||||
*/
|
||||
customStopWords?: string[];
|
||||
|
||||
/**
|
||||
* whether to remove punctuation
|
||||
*/
|
||||
@@ -572,6 +582,14 @@ export interface FtsOptions {
|
||||
* whether to only index the prefix of the token for ngram tokenizer
|
||||
*/
|
||||
prefixOnly?: boolean;
|
||||
|
||||
/**
|
||||
* Number of documents per compressed posting block.
|
||||
*
|
||||
* The default is 128. Supported values are 128 and 256. A value of 256 uses
|
||||
* the experimental FTS V3 format and may introduce breaking changes.
|
||||
*/
|
||||
blockSize?: 128 | 256;
|
||||
}
|
||||
|
||||
export class Index {
|
||||
@@ -747,10 +765,12 @@ export class Index {
|
||||
options?.lowercase,
|
||||
options?.stem,
|
||||
options?.removeStopWords,
|
||||
options?.customStopWords,
|
||||
options?.asciiFolding,
|
||||
options?.ngramMinLength,
|
||||
options?.ngramMaxLength,
|
||||
options?.prefixOnly,
|
||||
options?.blockSize,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.33.0-beta.0",
|
||||
"version": "0.32.0-beta.3",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
+14
-4
@@ -43,6 +43,7 @@ pub fn tokenize(
|
||||
lower_case: Option<bool>,
|
||||
stem: Option<bool>,
|
||||
remove_stop_words: Option<bool>,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: Option<bool>,
|
||||
ngram_min_length: Option<u32>,
|
||||
ngram_max_length: Option<u32>,
|
||||
@@ -72,6 +73,7 @@ pub fn tokenize(
|
||||
if let Some(remove_stop_words) = remove_stop_words {
|
||||
opts = opts.remove_stop_words(remove_stop_words);
|
||||
}
|
||||
opts = opts.custom_stop_words(custom_stop_words);
|
||||
if let Some(ascii_folding) = ascii_folding {
|
||||
opts = opts.ascii_folding(ascii_folding);
|
||||
}
|
||||
@@ -222,11 +224,13 @@ impl Index {
|
||||
lower_case: Option<bool>,
|
||||
stem: Option<bool>,
|
||||
remove_stop_words: Option<bool>,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: Option<bool>,
|
||||
ngram_min_length: Option<u32>,
|
||||
ngram_max_length: Option<u32>,
|
||||
prefix_only: Option<bool>,
|
||||
) -> Self {
|
||||
block_size: Option<u32>,
|
||||
) -> napi::Result<Self> {
|
||||
let mut opts = FtsIndexBuilder::default();
|
||||
if let Some(with_position) = with_position {
|
||||
opts = opts.with_position(with_position);
|
||||
@@ -249,6 +253,7 @@ impl Index {
|
||||
if let Some(remove_stop_words) = remove_stop_words {
|
||||
opts = opts.remove_stop_words(remove_stop_words);
|
||||
}
|
||||
opts = opts.custom_stop_words(custom_stop_words);
|
||||
if let Some(ascii_folding) = ascii_folding {
|
||||
opts = opts.ascii_folding(ascii_folding);
|
||||
}
|
||||
@@ -261,10 +266,15 @@ impl Index {
|
||||
if let Some(prefix_only) = prefix_only {
|
||||
opts = opts.ngram_prefix_only(prefix_only);
|
||||
}
|
||||
|
||||
Self {
|
||||
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
|
||||
if let Some(block_size) = block_size {
|
||||
opts = opts
|
||||
.block_size(block_size as usize)
|
||||
.map_err(|err| napi::Error::from_reason(err.to_string()))?;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(factory)]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.36.0"
|
||||
current_version = "0.35.0-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.36.0"
|
||||
version = "0.35.0-beta.3"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
|
||||
@@ -63,7 +63,7 @@ tests = [
|
||||
"polars>=0.19, <=1.3.0",
|
||||
"pyarrow<25",
|
||||
"pyarrow-stubs>=16.0",
|
||||
"pylance==9.0.0",
|
||||
"pylance==9.0.0rc1",
|
||||
"requests>=2.31.0",
|
||||
"datafusion>=54,<55",
|
||||
"opentelemetry-sdk>=1.30.0",
|
||||
|
||||
@@ -258,6 +258,7 @@ def tokenize(
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
@@ -265,9 +266,10 @@ def tokenize(
|
||||
) -> Iterable[FtsToken]:
|
||||
"""Tokenize a full-text search query using an explicit tokenizer.
|
||||
|
||||
This does not require a table or FTS index. The tokenizer options match
|
||||
:class:`lancedb.index.FTS`.
|
||||
This does not require an FTS index. The tokenizer options match
|
||||
:class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
|
||||
"""
|
||||
|
||||
return _tokenize(
|
||||
query,
|
||||
base_tokenizer=base_tokenizer,
|
||||
@@ -276,6 +278,7 @@ def tokenize(
|
||||
lower_case=lower_case,
|
||||
stem=stem,
|
||||
remove_stop_words=remove_stop_words,
|
||||
custom_stop_words=custom_stop_words,
|
||||
ascii_folding=ascii_folding,
|
||||
ngram_min_length=ngram_min_length,
|
||||
ngram_max_length=ngram_max_length,
|
||||
|
||||
@@ -59,6 +59,7 @@ def tokenize(
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, Optional
|
||||
from typing import List, Literal, Optional
|
||||
|
||||
from ._lancedb import (
|
||||
IndexConfig,
|
||||
@@ -115,6 +115,12 @@ class FTS:
|
||||
|
||||
For example, it works with `title`, `description`, `content`, etc.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Create an index configuration that uses 256-document posting blocks:
|
||||
|
||||
>>> config = FTS(block_size=256)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
with_position : bool, default False
|
||||
@@ -145,9 +151,18 @@ class FTS:
|
||||
remove_stop_words : bool, default True
|
||||
Whether to remove stop words. Stop words are common words that are often
|
||||
removed from text before indexing. For example, in English "the" and "and".
|
||||
custom_stop_words : list of str, optional
|
||||
Custom words replace the built-in language stop words
|
||||
and only take effect when ``remove_stop_words`` is True. ``None`` uses
|
||||
the built-in language list, while an empty list explicitly uses no
|
||||
stop words.
|
||||
ascii_folding : bool, default True
|
||||
Whether to fold ASCII characters. This converts accented characters to
|
||||
their ASCII equivalent. For example, "café" would be converted to "cafe".
|
||||
block_size : int, default 128
|
||||
The number of documents per compressed posting block. Supported values
|
||||
are 128 and 256. A value of 256 uses the experimental FTS V3 format
|
||||
and may introduce breaking changes.
|
||||
|
||||
Notes
|
||||
-----
|
||||
@@ -168,6 +183,8 @@ class FTS:
|
||||
ngram_min_length: int = 3
|
||||
ngram_max_length: int = 3
|
||||
prefix_only: bool = False
|
||||
block_size: int = 128
|
||||
custom_stop_words: Optional[List[str]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -340,10 +340,12 @@ class RemoteTable(Table):
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -360,10 +362,12 @@ class RemoteTable(Table):
|
||||
lower_case=lower_case,
|
||||
stem=stem,
|
||||
remove_stop_words=remove_stop_words,
|
||||
custom_stop_words=custom_stop_words,
|
||||
ascii_folding=ascii_folding,
|
||||
ngram_min_length=ngram_min_length,
|
||||
ngram_max_length=ngram_max_length,
|
||||
prefix_only=prefix_only,
|
||||
block_size=block_size,
|
||||
)
|
||||
LOOP.run(
|
||||
self._table.create_index(
|
||||
|
||||
@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
|
||||
column : str, default "text"
|
||||
The name of the column to use as input to the cross encoder model.
|
||||
return_score : str, default "relevance"
|
||||
options are "relevance" or "all". Only "relevance" is supported for now.
|
||||
options are "relevance" or "all".
|
||||
**kwargs
|
||||
Additional keyword arguments to pass to the model. For example, 'device'.
|
||||
See AnswerDotAI/rerankers for more information.
|
||||
@@ -77,12 +77,13 @@ class AnswerdotaiRerankers(Reranker):
|
||||
vector_results: pa.Table,
|
||||
fts_results: pa.Table,
|
||||
):
|
||||
combined_results = self.merge_results(vector_results, fts_results)
|
||||
if self.score == "all":
|
||||
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
|
||||
else:
|
||||
combined_results = self.merge_results(vector_results, fts_results)
|
||||
combined_results = self._rerank(combined_results, query)
|
||||
if self.score == "relevance":
|
||||
combined_results = self._keep_relevance_score(combined_results)
|
||||
elif self.score == "all":
|
||||
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
|
||||
combined_results = combined_results.sort_by(
|
||||
[("_relevance_score", "descending")]
|
||||
)
|
||||
|
||||
@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
|
||||
column : str, default "text"
|
||||
The name of the column to use as input to the cross encoder model.
|
||||
return_score : str, default "relevance"
|
||||
options are "relevance" or "all". Only "relevance" is supported for now.
|
||||
options are "relevance" or "all".
|
||||
**kwargs
|
||||
Additional keyword arguments to pass to the model, for example, 'device'.
|
||||
See AnswerDotAI/rerankers for more information.
|
||||
|
||||
@@ -1102,10 +1102,12 @@ class Table(ABC):
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
@@ -1168,6 +1170,9 @@ class Table(ABC):
|
||||
remove_stop_words : bool, default True
|
||||
Whether to remove stop words. Stop words are common words that are often
|
||||
removed from text before indexing. For example, in English "the" and "and".
|
||||
custom_stop_words : list of str, optional
|
||||
Custom words that replace the built-in language stop words. ``None``
|
||||
uses the built-in list; an empty list explicitly uses no stop words.
|
||||
ascii_folding : bool, default True
|
||||
Whether to fold ASCII characters. This converts accented characters to
|
||||
their ASCII equivalent. For example, "café" would be converted to "cafe".
|
||||
@@ -1177,6 +1182,10 @@ class Table(ABC):
|
||||
The maximum length of an n-gram.
|
||||
prefix_only: bool, default False
|
||||
Whether to only index the prefix of the token for ngram tokenizer.
|
||||
block_size: int, default 128
|
||||
The number of documents per compressed posting block. Must be 128
|
||||
or 256. A value of 256 uses the experimental FTS V3 format and
|
||||
may introduce breaking changes.
|
||||
wait_timeout: timedelta, optional
|
||||
The timeout to wait if indexing is asynchronous.
|
||||
name: str, optional
|
||||
@@ -3022,10 +3031,12 @@ class LanceTable(Table):
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -3067,6 +3078,7 @@ class LanceTable(Table):
|
||||
"lower_case": lower_case,
|
||||
"stem": stem,
|
||||
"remove_stop_words": remove_stop_words,
|
||||
"custom_stop_words": custom_stop_words,
|
||||
"ascii_folding": ascii_folding,
|
||||
"ngram_min_length": ngram_min_length,
|
||||
"ngram_max_length": ngram_max_length,
|
||||
@@ -3074,10 +3086,9 @@ class LanceTable(Table):
|
||||
}
|
||||
else:
|
||||
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
|
||||
tokenizer_configs["custom_stop_words"] = custom_stop_words
|
||||
|
||||
config = FTS(
|
||||
**tokenizer_configs,
|
||||
)
|
||||
config = FTS(block_size=block_size, **tokenizer_configs)
|
||||
|
||||
try:
|
||||
LOOP.run(
|
||||
|
||||
@@ -219,11 +219,48 @@ def test_create_inverted_index(table, with_position):
|
||||
table.create_fts_index(
|
||||
"text",
|
||||
with_position=with_position,
|
||||
custom_stop_words=["puppy"],
|
||||
name="custom_fts_index",
|
||||
)
|
||||
indices = table.list_indices()
|
||||
fts_indices = [i for i in indices if i.index_type == "FTS"]
|
||||
assert any(i.name == "custom_fts_index" for i in fts_indices)
|
||||
assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [128, 256])
|
||||
def test_create_inverted_index_block_size(table, block_size):
|
||||
table.create_index("text", config=FTS(block_size=block_size))
|
||||
|
||||
index = next(index for index in table.list_indices() if index.index_type == "FTS")
|
||||
assert index.index_details["block_size"] == block_size
|
||||
assert index.index_version == (2 if block_size == 128 else 3)
|
||||
|
||||
results = table.search("puppy").limit(5).to_list()
|
||||
assert len(results) == 5
|
||||
|
||||
|
||||
def test_create_inverted_index_rejects_invalid_block_size(table):
|
||||
with pytest.raises(ValueError, match="128 or 256"):
|
||||
table.create_index("text", config=FTS(block_size=129))
|
||||
|
||||
|
||||
def test_custom_stop_words_list(table):
|
||||
table.create_index(
|
||||
"text",
|
||||
config=FTS(stem=False, custom_stop_words=["lance"]),
|
||||
)
|
||||
|
||||
assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"]
|
||||
tokens = table.tokenize("the lance data", column="text")
|
||||
assert [token.text for token in tokens] == ["the", "data"]
|
||||
empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[])
|
||||
assert [token.text for token in empty_tokens] == ["the", "lance", "data"]
|
||||
with pytest.raises(TypeError, match=r"custom_stop_words.*int"):
|
||||
ldb.tokenize(
|
||||
"the lance data",
|
||||
custom_stop_words=["lance", 42],
|
||||
)
|
||||
|
||||
|
||||
def test_search_fts(table):
|
||||
|
||||
@@ -768,7 +768,11 @@ def test_table_create_indices():
|
||||
# Test create_fts_index with custom name (legacy method)
|
||||
with pytest.warns(DeprecationWarning, match="create_fts_index"):
|
||||
table.create_fts_index(
|
||||
"text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
|
||||
"text",
|
||||
wait_timeout=timedelta(seconds=2),
|
||||
block_size=256,
|
||||
custom_stop_words=["cloud"],
|
||||
name="custom_fts_idx",
|
||||
)
|
||||
|
||||
# Test create_index with custom name (legacy form: vector_column_name kwarg)
|
||||
@@ -791,6 +795,8 @@ def test_table_create_indices():
|
||||
fts_req = received_requests[1]
|
||||
assert "name" in fts_req
|
||||
assert fts_req["name"] == "custom_fts_idx"
|
||||
assert fts_req["block_size"] == 256
|
||||
assert fts_req["custom_stop_words"] == ["cloud"]
|
||||
|
||||
# Check vector index request has custom name
|
||||
vector_req = received_requests[2]
|
||||
@@ -876,7 +882,7 @@ def test_remote_create_index_new_api():
|
||||
_warnings.simplefilter("error", DeprecationWarning)
|
||||
table.create_index("vector", config=IvfPq(distance_type="l2"))
|
||||
table.create_index("category", config=BTree())
|
||||
table.create_index("text", config=FTS())
|
||||
table.create_index("text", config=FTS(block_size=256))
|
||||
# IvfRq via new API
|
||||
table.create_index("vector", config=IvfRq(distance_type="l2"))
|
||||
|
||||
@@ -896,6 +902,7 @@ def test_remote_create_index_new_api():
|
||||
"vector",
|
||||
"vector",
|
||||
]
|
||||
assert received_requests[2]["block_size"] == 256
|
||||
|
||||
|
||||
def test_table_wait_for_index_timeout():
|
||||
|
||||
@@ -644,6 +644,21 @@ def test_cross_encoder_reranker_return_all(tmp_path):
|
||||
assert "_distance" in result.column_names
|
||||
|
||||
|
||||
def test_answerdotai_reranker_return_all(tmp_path):
|
||||
pytest.importorskip("rerankers")
|
||||
reranker = AnswerdotaiRerankers(return_score="all")
|
||||
table, schema = get_test_table(tmp_path)
|
||||
query = "single player experience"
|
||||
result = (
|
||||
table.search(query, query_type="hybrid", vector_column_name="vector")
|
||||
.rerank(reranker=reranker)
|
||||
.to_arrow()
|
||||
)
|
||||
assert "_relevance_score" in result.column_names
|
||||
assert "_score" in result.column_names
|
||||
assert "_distance" in result.column_names
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regression tests for LinearCombinationReranker scoring bugs (issue #3154)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+7
-1
@@ -59,7 +59,11 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
|
||||
.ascii_folding(params.ascii_folding)
|
||||
.ngram_min_length(params.ngram_min_length)
|
||||
.ngram_max_length(params.ngram_max_length)
|
||||
.ngram_prefix_only(params.prefix_only);
|
||||
.ngram_prefix_only(params.prefix_only)
|
||||
.custom_stop_words(params.custom_stop_words);
|
||||
let inner_opts = inner_opts
|
||||
.block_size(params.block_size)
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))?;
|
||||
Ok(LanceDbIndex::FTS(inner_opts))
|
||||
}
|
||||
"IvfFlat" => {
|
||||
@@ -203,10 +207,12 @@ struct FtsParams {
|
||||
lower_case: bool,
|
||||
stem: bool,
|
||||
remove_stop_words: bool,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: bool,
|
||||
ngram_min_length: u32,
|
||||
ngram_max_length: u32,
|
||||
prefix_only: bool,
|
||||
block_size: usize,
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
|
||||
+4
-1
@@ -520,6 +520,7 @@ impl From<LanceDbFtsToken> for FtsToken {
|
||||
lower_case = true,
|
||||
stem = true,
|
||||
remove_stop_words = true,
|
||||
custom_stop_words = None,
|
||||
ascii_folding = true,
|
||||
ngram_min_length = 3,
|
||||
ngram_max_length = 3,
|
||||
@@ -534,6 +535,7 @@ pub fn tokenize(
|
||||
lower_case: bool,
|
||||
stem: bool,
|
||||
remove_stop_words: bool,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: bool,
|
||||
ngram_min_length: u32,
|
||||
ngram_max_length: u32,
|
||||
@@ -555,7 +557,8 @@ pub fn tokenize(
|
||||
.ascii_folding(ascii_folding)
|
||||
.ngram_min_length(ngram_min_length)
|
||||
.ngram_max_length(ngram_max_length)
|
||||
.ngram_prefix_only(prefix_only);
|
||||
.ngram_prefix_only(prefix_only)
|
||||
.custom_stop_words(custom_stop_words);
|
||||
let tokens = lancedb_tokenize(&query, ¶ms).infer_error()?;
|
||||
Ok(tokens.into_iter().map(FtsToken::from).collect())
|
||||
}
|
||||
|
||||
Generated
+9
-9
@@ -1998,7 +1998,7 @@ requires-dist = [
|
||||
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
|
||||
{ name = "pydantic", specifier = ">=1.10" },
|
||||
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
|
||||
{ name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0" },
|
||||
{ name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" },
|
||||
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
|
||||
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
|
||||
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
|
||||
@@ -3854,8 +3854,8 @@ crypto = [
|
||||
|
||||
[[package]]
|
||||
name = "pylance"
|
||||
version = "9.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
version = "9.0.0rc1"
|
||||
source = { registry = "https://pypi.fury.io/lance-format" }
|
||||
dependencies = [
|
||||
{ name = "lance-namespace" },
|
||||
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
@@ -3863,12 +3863,12 @@ dependencies = [
|
||||
{ name = "pyarrow" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/be/45733acd64801991852aac8e658601fd8fc12f76ceb81e57fca690896b90/pylance-9.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8257213501d3298c5b6a344d60938e4bbe4de9f00cd3265371a56d1dc3dd15ca", size = 68377982, upload-time = "2026-07-24T16:53:45.247Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d0/3e/1ef707cb215cc7268c63ad84a91344ad6313d3984b343eeb19a9b708698e/pylance-9.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:804eedfa1fda2e703cca8580c76f0b44a1b849b725a8908c06f8acfca811732f", size = 71844362, upload-time = "2026-07-24T16:56:07.6Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/fb/a499e5c53ddb75c7de44100fd2bb1f7cc735966200d20292eed7af9ef552/pylance-9.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a0b75595e3766c1d5f4c90abdc52337b4de60f1a52d123a4f8c4e5bcbbbfa8f", size = 75663088, upload-time = "2026-07-24T17:10:31.283Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/80/0714e09f64a68dbdf62558955e737a7436df763b9834a6aba506861b5352/pylance-9.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d2f69c5c390ae3710c35a429905fe15f769951777fafb1b72a359e035ec121f7", size = 71866858, upload-time = "2026-07-24T16:56:35.937Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/4b/3c/78d3a6d6ca0d843b7c3ac0c30d9cd2cf4635b0c2cabe6ec66583d5bfc1a1/pylance-9.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:836268a7832d62f3d5ccbe1c3fca239621971d90297bcfff14a70b3cb6842aa8", size = 75642656, upload-time = "2026-07-24T17:13:11.879Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/c1/dc9c9a31e171530ec0add024d922488c046437d32a7087c29e254a7eacc7/pylance-9.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:96441d27a5ed3805300388ccf8f31835cd280c98751f80b6a1a11dcd6808fc43", size = 81668288, upload-time = "2026-07-24T17:05:38.707Z" },
|
||||
{ url = "https://pypi.fury.io/lance-format/-/ver_vEHBE/pylance-9.0.0rc1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0b6b02a1808bb3072ee7fe4e36614cae6f86302513e73ec7f55b2234a963b24" },
|
||||
{ url = "https://pypi.fury.io/lance-format/-/ver_1Jipm4/pylance-9.0.0rc1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30f0ebf0d88034301819eb964f9236ce555aaa58e7ab89c5975a3e2250bbb405" },
|
||||
{ url = "https://pypi.fury.io/lance-format/-/ver_IvKxo/pylance-9.0.0rc1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44609ea2615ea6e684b85478d1694af2026458f61cf7895ecc75e238bfd17aa8" },
|
||||
{ url = "https://pypi.fury.io/lance-format/-/ver_2hidj1/pylance-9.0.0rc1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:182167a8dba9eeabffbffd53bd5b8548613d4d459b7cd7b34a840dd00cbb806f" },
|
||||
{ url = "https://pypi.fury.io/lance-format/-/ver_1dFx3r/pylance-9.0.0rc1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8a63b11e814b7eab758bcaf0d6f97eb05ea86203d9fb0af718c462c24c7d6c9c" },
|
||||
{ url = "https://pypi.fury.io/lance-format/-/ver_2a8dSh/pylance-9.0.0rc1-cp310-abi3-win_amd64.whl", hash = "sha256:2ff8b953ae2b0550490c1a7efd210aa91bc223d200ffac28849056cfd7436d97" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.33.0-beta.0"
|
||||
version = "0.32.0-beta.3"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
|
||||
@@ -76,7 +76,12 @@ async fn create_table(db: &Connection) -> Result<Table> {
|
||||
|
||||
async fn create_index(table: &Table) -> Result<()> {
|
||||
table
|
||||
.create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
|
||||
.create_index(
|
||||
&["doc"],
|
||||
Index::FTS(
|
||||
FtsIndexBuilder::default().custom_stop_words(Some(vec!["example".to_owned()])),
|
||||
),
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
Ok(())
|
||||
|
||||
@@ -273,7 +273,11 @@ pub(crate) async fn take_blobs_aligned(
|
||||
if *is_null {
|
||||
builder.append_null();
|
||||
} else {
|
||||
builder.append_value(payloads[payload_idx].data.as_ref());
|
||||
if let Some(data) = &payloads[payload_idx].data {
|
||||
builder.append_value(data);
|
||||
} else {
|
||||
builder.append_null();
|
||||
}
|
||||
payload_idx += 1;
|
||||
}
|
||||
}
|
||||
@@ -315,7 +319,7 @@ pub(crate) async fn take_blob_files_aligned(
|
||||
if *is_null {
|
||||
None
|
||||
} else {
|
||||
Some(handles.next().unwrap())
|
||||
handles.next().flatten()
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
|
||||
@@ -54,7 +54,26 @@ pub enum Index {
|
||||
/// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
|
||||
Fm(FmIndexBuilder),
|
||||
|
||||
/// Full text search index using bm25.
|
||||
/// Full text search index using BM25.
|
||||
///
|
||||
/// The posting block size defaults to 128. Supported values are 128 and 256;
|
||||
/// a value of 256 uses the experimental FTS V3 format and may introduce
|
||||
/// breaking changes.
|
||||
///
|
||||
/// ```
|
||||
/// use lancedb::index::{Index, scalar::FtsIndexBuilder};
|
||||
///
|
||||
/// # async fn create_fts_index(
|
||||
/// # table: &lancedb::Table,
|
||||
/// # ) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let params = FtsIndexBuilder::default().block_size(256)?;
|
||||
/// table
|
||||
/// .create_index(&["text"], Index::FTS(params))
|
||||
/// .execute()
|
||||
/// .await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
FTS(FtsIndexBuilder),
|
||||
|
||||
/// IVF index
|
||||
|
||||
@@ -4496,6 +4496,28 @@ mod tests {
|
||||
serde_json::to_value(InvertedIndexParams::default()).unwrap(),
|
||||
Index::FTS(Default::default()),
|
||||
),
|
||||
(
|
||||
"FTS",
|
||||
{
|
||||
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
|
||||
body["block_size"] = 256.into();
|
||||
body
|
||||
},
|
||||
Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
|
||||
),
|
||||
(
|
||||
"FTS",
|
||||
{
|
||||
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
|
||||
body["custom_stop_words"] = json!(["cat", " cat ", "CAT"]);
|
||||
body
|
||||
},
|
||||
Index::FTS(InvertedIndexParams::default().custom_stop_words(Some(vec![
|
||||
"cat".to_string(),
|
||||
" cat ".to_string(),
|
||||
"CAT".to_string(),
|
||||
]))),
|
||||
),
|
||||
];
|
||||
|
||||
for (index_type, expected_body, index) in cases {
|
||||
@@ -5027,8 +5049,9 @@ mod tests {
|
||||
"max_token_length": 40,
|
||||
"lower_case": true,
|
||||
"stem": false,
|
||||
"remove_stop_words": false,
|
||||
"remove_stop_words": true,
|
||||
"ascii_folding": true,
|
||||
"custom_stop_words": ["hello"],
|
||||
})
|
||||
.to_string();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
@@ -5066,10 +5089,6 @@ mod tests {
|
||||
assert_eq!(
|
||||
tokens,
|
||||
vec![
|
||||
FtsToken {
|
||||
text: "hello".to_string(),
|
||||
position: 0,
|
||||
},
|
||||
FtsToken {
|
||||
text: "こんにちは".to_string(),
|
||||
position: 1,
|
||||
|
||||
@@ -21,6 +21,7 @@ use lance::dataset::WriteMode;
|
||||
use lance::dataset::builder::DatasetBuilder;
|
||||
use lance::dataset::{InsertBuilder, WriteParams};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance::index::scalar::load_segment_params;
|
||||
use lance::io::{ObjectStoreParams, WrappingObjectStore};
|
||||
use lance_datafusion::utils::StreamingWriteSource;
|
||||
use lance_index::IndexCriteria;
|
||||
@@ -3131,10 +3132,9 @@ impl BaseTable for NativeTable {
|
||||
async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
|
||||
let dataset = self.dataset.get().await?;
|
||||
let total_rows = dataset.count_rows(None).await? as u64;
|
||||
let indices = dataset
|
||||
.describe_indices(None)
|
||||
.await?
|
||||
.into_iter()
|
||||
let descriptions = dataset.describe_indices(None).await?;
|
||||
let mut indices: Vec<IndexConfig> = descriptions
|
||||
.iter()
|
||||
.filter_map(|idx_desc| {
|
||||
let index_type: crate::index::IndexType = idx_desc
|
||||
.index_type()
|
||||
@@ -3192,6 +3192,31 @@ impl BaseTable for NativeTable {
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for index in indices
|
||||
.iter_mut()
|
||||
.filter(|index| index.index_type == crate::index::IndexType::FTS)
|
||||
{
|
||||
let Some(description) = descriptions
|
||||
.iter()
|
||||
.find(|description| description.name() == index.name)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let segments = description.segments();
|
||||
let Some(segment) = segments.first() else {
|
||||
continue;
|
||||
};
|
||||
let params = load_segment_params(&dataset, segment).await?;
|
||||
let details = serde_json::to_string(¶ms).map_err(|source| Error::Other {
|
||||
message: format!(
|
||||
"Failed to serialize full text search configuration for index '{}'",
|
||||
index.name
|
||||
),
|
||||
source: Some(Box::new(source)),
|
||||
})?;
|
||||
index.index_details = Some(details);
|
||||
}
|
||||
Ok(indices)
|
||||
}
|
||||
|
||||
|
||||
@@ -382,7 +382,9 @@ mod tests {
|
||||
use crate::connect;
|
||||
use crate::connection::ConnectBuilder;
|
||||
use crate::index::Index;
|
||||
use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder};
|
||||
use crate::index::scalar::{
|
||||
BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
|
||||
};
|
||||
use crate::index::vector::{
|
||||
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
|
||||
};
|
||||
@@ -1362,16 +1364,57 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
table
|
||||
.create_index(&["text"], Index::FTS(Default::default()))
|
||||
.create_index(
|
||||
&["text"],
|
||||
Index::FTS(
|
||||
FtsIndexBuilder::default()
|
||||
.stem(false)
|
||||
.custom_stop_words(Some(vec!["cat".to_string()]))
|
||||
.block_size(256)
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
drop(table);
|
||||
let table = conn.open_table("test_bitmap").execute().await.unwrap();
|
||||
let index_configs = table.list_indices().await.unwrap();
|
||||
assert_eq!(index_configs.len(), 1);
|
||||
let index = index_configs.into_iter().next().unwrap();
|
||||
assert_eq!(index.index_type, crate::index::IndexType::FTS);
|
||||
assert_eq!(index.columns, vec!["text".to_string()]);
|
||||
assert_eq!(index.name, "text_idx");
|
||||
assert_eq!(index.index_version, Some(3));
|
||||
let index_params: FtsIndexBuilder =
|
||||
serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(index_params.posting_block_size(), 256);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&index_params).unwrap()["custom_stop_words"],
|
||||
serde_json::json!(["cat"])
|
||||
);
|
||||
assert_eq!(
|
||||
table
|
||||
.tokenize("cat dog", "text_idx")
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|token| token.text)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["dog"]
|
||||
);
|
||||
|
||||
let batches = table
|
||||
.query()
|
||||
.full_text_search(FullTextSearchQuery::new("cat dog".to_string()))
|
||||
.limit(120)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 40);
|
||||
|
||||
let num_rows = 120;
|
||||
let stats = table.index_stats("text_idx").await.unwrap().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user