mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-28 17:08:43 +00:00
Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Yang Cen
|
98945e55e5 | ||
 lancedb automation
|
5eedeea120 | ||
 Lance Release
|
a00edef0e6 | ||
|
Lance Release
|
1b2670443e | ||
|
Yang Cen
|
9dc5ec03aa | ||
 Andrew Chen
|
18760f74cd | ||
|
LanceDB Robot
|
c9d07ef6fc |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.32.0-beta.2"
|
||||
current_version = "0.32.0-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+58
-45
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4852,8 +4852,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4875,7 +4875,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4889,7 +4889,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4898,8 +4898,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4909,8 +4909,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4933,6 +4933,7 @@ dependencies = [
|
||||
"object_store",
|
||||
"pin-project",
|
||||
"prost",
|
||||
"quick_cache",
|
||||
"rand 0.9.5",
|
||||
"roaring",
|
||||
"serde_json",
|
||||
@@ -4948,8 +4949,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4979,8 +4980,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4997,8 +4998,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5007,8 +5008,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5043,8 +5044,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5074,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5142,8 +5143,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5165,8 +5166,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -5209,8 +5210,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5226,8 +5227,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5239,8 +5240,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5294,8 +5295,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5310,8 +5311,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5350,8 +5351,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5364,8 +5365,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "9.1.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
|
||||
version = "10.0.0-beta.5"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v10.0.0-beta.5#ddb8e28ca238f29628b8e1795ddccbb7bf75e5c8"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
@@ -5378,7 +5379,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.32.0-beta.2"
|
||||
version = "0.32.0-beta.3"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5466,7 +5467,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.32.0-beta.2"
|
||||
version = "0.32.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5491,7 +5492,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.35.0-beta.2"
|
||||
version = "0.35.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -7802,6 +7803,18 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=10.0.0-beta.5", default-features = false, "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=10.0.0-beta.5", "tag" = "v10.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
|
||||
+11
-1
@@ -453,6 +453,16 @@ paths:
|
||||
The metric type to use for the index. l2, Cosine, Dot are supported.
|
||||
index_type:
|
||||
type: string
|
||||
custom_stop_words:
|
||||
type: [array, "null"]
|
||||
items:
|
||||
type: string
|
||||
description: |
|
||||
The custom stop-word list for an FTS index. A non-null
|
||||
array replaces the language's built-in stop-word list and is only
|
||||
applied when remove_stop_words is enabled. Null uses the built-in
|
||||
language list, while an empty array explicitly replaces it with no
|
||||
stop words.
|
||||
responses:
|
||||
"200":
|
||||
description: Index successfully created
|
||||
@@ -510,4 +520,4 @@ paths:
|
||||
"401":
|
||||
$ref: "#/components/responses/unauthorized"
|
||||
"404":
|
||||
$ref: "#/components/responses/not_found"
|
||||
$ref: "#/components/responses/not_found"
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.32.0-beta.2</version>
|
||||
<version>0.32.0-beta.3</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -43,6 +43,34 @@ The following tokenizers are available:
|
||||
|
||||
***
|
||||
|
||||
### blockSize?
|
||||
|
||||
```ts
|
||||
optional blockSize: 128 | 256;
|
||||
```
|
||||
|
||||
Number of documents per compressed posting block.
|
||||
|
||||
The default is 128. Supported values are 128 and 256. A value of 256 uses
|
||||
the experimental FTS V3 format and may introduce breaking changes.
|
||||
|
||||
***
|
||||
|
||||
### customStopWords?
|
||||
|
||||
```ts
|
||||
optional customStopWords: string[];
|
||||
```
|
||||
|
||||
Custom stop words that replace the built-in list for `language`.
|
||||
|
||||
This option only affects tokenization when `removeStopWords` is true.
|
||||
|
||||
`undefined` keeps the built-in language list. An empty array explicitly
|
||||
replaces it with no stop words.
|
||||
|
||||
***
|
||||
|
||||
### language?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -30,6 +30,21 @@ The tokenizer to use. The default is "simple".
|
||||
|
||||
***
|
||||
|
||||
### customStopWords?
|
||||
|
||||
```ts
|
||||
optional customStopWords: string[];
|
||||
```
|
||||
|
||||
Custom stop words that replace the built-in list for `language`.
|
||||
|
||||
This option only affects tokenization when `removeStopWords` is true.
|
||||
|
||||
`undefined` keeps the built-in language list. An empty array explicitly
|
||||
replaces it with no stop words.
|
||||
|
||||
***
|
||||
|
||||
### language?
|
||||
|
||||
```ts
|
||||
|
||||
@@ -94,9 +94,33 @@ of raw SQL strings with [where][lancedb.query.LanceQueryBuilder.where] and
|
||||
|
||||
## Full text search
|
||||
|
||||
Use [lancedb.table.Table.create_fts_index][] for the synchronous API or
|
||||
[lancedb.table.AsyncTable.create_index][] with [lancedb.index.FTS][] for the
|
||||
asynchronous API.
|
||||
Pass `custom_stop_words` to [lancedb.index.FTS][]:
|
||||
|
||||
```python
|
||||
from lancedb.index import FTS
|
||||
|
||||
table.create_index(
|
||||
"text",
|
||||
config=FTS(remove_stop_words=True, custom_stop_words=["acme", "internal"]),
|
||||
)
|
||||
```
|
||||
|
||||
The list replaces the built-in stop words and is used only when
|
||||
`remove_stop_words=True`:
|
||||
|
||||
- `custom_stop_words=None` uses the built-in list for `language`.
|
||||
- `custom_stop_words=[]` removes no words.
|
||||
- Values are passed through without trimming, lowercasing, or other rewriting.
|
||||
|
||||
The same option is available on `lancedb.tokenize(...)` and the deprecated
|
||||
[lancedb.table.Table.create_fts_index][] compatibility helper:
|
||||
|
||||
```python
|
||||
import lancedb
|
||||
|
||||
tokens = list(lancedb.tokenize("acme makes searchable data",
|
||||
custom_stop_words=["acme"]))
|
||||
```
|
||||
|
||||
::: lancedb.index.FTS
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.32.0-beta.2</version>
|
||||
<version>0.32.0-beta.3</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.32.0-beta.2</version>
|
||||
<version>0.32.0-beta.3</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>9.1.0-beta.8</lance-core.version>
|
||||
<lance-core.version>10.0.0-beta.5</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.32.0-beta.2"
|
||||
version = "0.32.0-beta.3"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
OAuthHeaderProvider,
|
||||
StaticHeaderProvider,
|
||||
} from "../lancedb/header";
|
||||
import { Index } from "../lancedb/indices";
|
||||
|
||||
// Test-only header providers
|
||||
class CustomProvider extends HeaderProvider {
|
||||
@@ -225,6 +226,59 @@ describe("remote connection", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("sends FTS options to remote tables", async () => {
|
||||
let createIndexBody: Record<string, unknown> | undefined;
|
||||
|
||||
await withMockDatabase(
|
||||
(req, res) => {
|
||||
const path = req.url ?? "";
|
||||
if (path.endsWith("/describe/")) {
|
||||
res.writeHead(200, { "Content-Type": "application/json" }).end(
|
||||
JSON.stringify({
|
||||
name: "t",
|
||||
version: 1,
|
||||
schema: {
|
||||
fields: [
|
||||
{ name: "text", type: { type: "string" }, nullable: false },
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (path.endsWith("/create_index/")) {
|
||||
let raw = "";
|
||||
req.on("data", (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
createIndexBody = JSON.parse(raw);
|
||||
res.writeHead(200).end();
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
res.writeHead(404).end();
|
||||
},
|
||||
async (db) => {
|
||||
const table = await db.openTable("t");
|
||||
await table.createIndex("text", {
|
||||
config: Index.fts({
|
||||
blockSize: 256,
|
||||
removeStopWords: true,
|
||||
customStopWords: ["the"],
|
||||
}),
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
expect(createIndexBody?.["column"]).toBe("text");
|
||||
expect(createIndexBody?.["index_type"]).toBe("FTS");
|
||||
expect(createIndexBody?.["block_size"]).toBe(256);
|
||||
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
|
||||
});
|
||||
|
||||
it("diffs and merges remote branches", async () => {
|
||||
const sampleDiff = {
|
||||
fromBranch: "exp",
|
||||
|
||||
@@ -2527,6 +2527,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
expect(results3.length).toBe(1);
|
||||
});
|
||||
|
||||
test("full text search with custom posting block size", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
{ text: "hello world", vector: [0.1, 0.2, 0.3] },
|
||||
{ text: "goodbye world", vector: [0.4, 0.5, 0.6] },
|
||||
];
|
||||
const table = await db.createTable("test", data);
|
||||
await table.createIndex("text", {
|
||||
config: Index.fts({ blockSize: 256 }),
|
||||
});
|
||||
|
||||
const index = (await table.listIndices()).find(
|
||||
(index) => index.indexType === "FTS",
|
||||
);
|
||||
expect(index?.indexVersion).toBe(3);
|
||||
expect(
|
||||
(index?.indexDetails as Record<string, unknown>)["block_size"],
|
||||
).toBe(256);
|
||||
|
||||
const results = await table.search("hello").toArray();
|
||||
expect(results[0].text).toBe(data[0].text);
|
||||
});
|
||||
|
||||
test("rejects invalid full text posting block size", () => {
|
||||
expect(() => Index.fts({ blockSize: 129 as 128 | 256 })).toThrow(
|
||||
"128 or 256",
|
||||
);
|
||||
});
|
||||
|
||||
test("full text search without lowercase", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = [
|
||||
@@ -2732,6 +2761,15 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
},
|
||||
);
|
||||
|
||||
test("tokenize supports custom stop words", async () => {
|
||||
const tokens = await tokenize("the lance data", {
|
||||
stem: false,
|
||||
removeStopWords: true,
|
||||
customStopWords: ["lance"],
|
||||
});
|
||||
expect(tokens.map((token) => token.text)).toEqual(["the", "data"]);
|
||||
});
|
||||
|
||||
describe("when calling explainPlan", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
|
||||
@@ -29,8 +29,14 @@ test("full text search", async () => {
|
||||
const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
|
||||
|
||||
await tbl.createIndex("doc", {
|
||||
config: lancedb.Index.fts(),
|
||||
config: lancedb.Index.fts({
|
||||
stem: false,
|
||||
removeStopWords: true,
|
||||
customStopWords: ["banana"],
|
||||
}),
|
||||
});
|
||||
const tokens = await tbl.tokenize("apple banana", { column: "doc" });
|
||||
expect(tokens.map((token) => token.text)).toEqual(["apple"]);
|
||||
|
||||
// --8<-- [start:full_text_search]
|
||||
const result = await tbl
|
||||
|
||||
@@ -194,6 +194,16 @@ export interface TokenizeOptions {
|
||||
/** Whether to remove stop words. */
|
||||
removeStopWords?: boolean;
|
||||
|
||||
/**
|
||||
* Custom stop words that replace the built-in list for `language`.
|
||||
*
|
||||
* This option only affects tokenization when `removeStopWords` is true.
|
||||
*
|
||||
* `undefined` keeps the built-in language list. An empty array explicitly
|
||||
* replaces it with no stop words.
|
||||
*/
|
||||
customStopWords?: string[];
|
||||
|
||||
/** Whether to fold ASCII characters. */
|
||||
asciiFolding?: boolean;
|
||||
|
||||
@@ -225,6 +235,7 @@ export async function tokenize(
|
||||
options?.lowercase,
|
||||
options?.stem,
|
||||
options?.removeStopWords,
|
||||
options?.customStopWords,
|
||||
options?.asciiFolding,
|
||||
options?.ngramMinLength,
|
||||
options?.ngramMaxLength,
|
||||
|
||||
@@ -553,6 +553,16 @@ export interface FtsOptions {
|
||||
*/
|
||||
removeStopWords?: boolean;
|
||||
|
||||
/**
|
||||
* Custom stop words that replace the built-in list for `language`.
|
||||
*
|
||||
* This option only affects tokenization when `removeStopWords` is true.
|
||||
*
|
||||
* `undefined` keeps the built-in language list. An empty array explicitly
|
||||
* replaces it with no stop words.
|
||||
*/
|
||||
customStopWords?: string[];
|
||||
|
||||
/**
|
||||
* whether to remove punctuation
|
||||
*/
|
||||
@@ -572,6 +582,14 @@ export interface FtsOptions {
|
||||
* whether to only index the prefix of the token for ngram tokenizer
|
||||
*/
|
||||
prefixOnly?: boolean;
|
||||
|
||||
/**
|
||||
* Number of documents per compressed posting block.
|
||||
*
|
||||
* The default is 128. Supported values are 128 and 256. A value of 256 uses
|
||||
* the experimental FTS V3 format and may introduce breaking changes.
|
||||
*/
|
||||
blockSize?: 128 | 256;
|
||||
}
|
||||
|
||||
export class Index {
|
||||
@@ -747,10 +765,12 @@ export class Index {
|
||||
options?.lowercase,
|
||||
options?.stem,
|
||||
options?.removeStopWords,
|
||||
options?.customStopWords,
|
||||
options?.asciiFolding,
|
||||
options?.ngramMinLength,
|
||||
options?.ngramMaxLength,
|
||||
options?.prefixOnly,
|
||||
options?.blockSize,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.32.0-beta.2",
|
||||
"version": "0.32.0-beta.3",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
+14
-4
@@ -43,6 +43,7 @@ pub fn tokenize(
|
||||
lower_case: Option<bool>,
|
||||
stem: Option<bool>,
|
||||
remove_stop_words: Option<bool>,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: Option<bool>,
|
||||
ngram_min_length: Option<u32>,
|
||||
ngram_max_length: Option<u32>,
|
||||
@@ -72,6 +73,7 @@ pub fn tokenize(
|
||||
if let Some(remove_stop_words) = remove_stop_words {
|
||||
opts = opts.remove_stop_words(remove_stop_words);
|
||||
}
|
||||
opts = opts.custom_stop_words(custom_stop_words);
|
||||
if let Some(ascii_folding) = ascii_folding {
|
||||
opts = opts.ascii_folding(ascii_folding);
|
||||
}
|
||||
@@ -222,11 +224,13 @@ impl Index {
|
||||
lower_case: Option<bool>,
|
||||
stem: Option<bool>,
|
||||
remove_stop_words: Option<bool>,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: Option<bool>,
|
||||
ngram_min_length: Option<u32>,
|
||||
ngram_max_length: Option<u32>,
|
||||
prefix_only: Option<bool>,
|
||||
) -> Self {
|
||||
block_size: Option<u32>,
|
||||
) -> napi::Result<Self> {
|
||||
let mut opts = FtsIndexBuilder::default();
|
||||
if let Some(with_position) = with_position {
|
||||
opts = opts.with_position(with_position);
|
||||
@@ -249,6 +253,7 @@ impl Index {
|
||||
if let Some(remove_stop_words) = remove_stop_words {
|
||||
opts = opts.remove_stop_words(remove_stop_words);
|
||||
}
|
||||
opts = opts.custom_stop_words(custom_stop_words);
|
||||
if let Some(ascii_folding) = ascii_folding {
|
||||
opts = opts.ascii_folding(ascii_folding);
|
||||
}
|
||||
@@ -261,10 +266,15 @@ impl Index {
|
||||
if let Some(prefix_only) = prefix_only {
|
||||
opts = opts.ngram_prefix_only(prefix_only);
|
||||
}
|
||||
|
||||
Self {
|
||||
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
|
||||
if let Some(block_size) = block_size {
|
||||
opts = opts
|
||||
.block_size(block_size as usize)
|
||||
.map_err(|err| napi::Error::from_reason(err.to_string()))?;
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),
|
||||
})
|
||||
}
|
||||
|
||||
#[napi(factory)]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.35.0-beta.2"
|
||||
current_version = "0.35.0-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.35.0-beta.2"
|
||||
version = "0.35.0-beta.3"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
|
||||
@@ -258,6 +258,7 @@ def tokenize(
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
@@ -265,9 +266,10 @@ def tokenize(
|
||||
) -> Iterable[FtsToken]:
|
||||
"""Tokenize a full-text search query using an explicit tokenizer.
|
||||
|
||||
This does not require a table or FTS index. The tokenizer options match
|
||||
:class:`lancedb.index.FTS`.
|
||||
This does not require an FTS index. The tokenizer options match
|
||||
:class:`lancedb.index.FTS`. ``custom_stop_words`` accepts a list of strings.
|
||||
"""
|
||||
|
||||
return _tokenize(
|
||||
query,
|
||||
base_tokenizer=base_tokenizer,
|
||||
@@ -276,6 +278,7 @@ def tokenize(
|
||||
lower_case=lower_case,
|
||||
stem=stem,
|
||||
remove_stop_words=remove_stop_words,
|
||||
custom_stop_words=custom_stop_words,
|
||||
ascii_folding=ascii_folding,
|
||||
ngram_min_length=ngram_min_length,
|
||||
ngram_max_length=ngram_max_length,
|
||||
|
||||
@@ -59,6 +59,7 @@ def tokenize(
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, Optional
|
||||
from typing import List, Literal, Optional
|
||||
|
||||
from ._lancedb import (
|
||||
IndexConfig,
|
||||
@@ -115,6 +115,12 @@ class FTS:
|
||||
|
||||
For example, it works with `title`, `description`, `content`, etc.
|
||||
|
||||
Examples
|
||||
--------
|
||||
Create an index configuration that uses 256-document posting blocks:
|
||||
|
||||
>>> config = FTS(block_size=256)
|
||||
|
||||
Attributes
|
||||
----------
|
||||
with_position : bool, default False
|
||||
@@ -145,9 +151,18 @@ class FTS:
|
||||
remove_stop_words : bool, default True
|
||||
Whether to remove stop words. Stop words are common words that are often
|
||||
removed from text before indexing. For example, in English "the" and "and".
|
||||
custom_stop_words : list of str, optional
|
||||
Custom words replace the built-in language stop words
|
||||
and only take effect when ``remove_stop_words`` is True. ``None`` uses
|
||||
the built-in language list, while an empty list explicitly uses no
|
||||
stop words.
|
||||
ascii_folding : bool, default True
|
||||
Whether to fold ASCII characters. This converts accented characters to
|
||||
their ASCII equivalent. For example, "café" would be converted to "cafe".
|
||||
block_size : int, default 128
|
||||
The number of documents per compressed posting block. Supported values
|
||||
are 128 and 256. A value of 256 uses the experimental FTS V3 format
|
||||
and may introduce breaking changes.
|
||||
|
||||
Notes
|
||||
-----
|
||||
@@ -168,6 +183,8 @@ class FTS:
|
||||
ngram_min_length: int = 3
|
||||
ngram_max_length: int = 3
|
||||
prefix_only: bool = False
|
||||
block_size: int = 128
|
||||
custom_stop_words: Optional[List[str]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -340,10 +340,12 @@ class RemoteTable(Table):
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -360,10 +362,12 @@ class RemoteTable(Table):
|
||||
lower_case=lower_case,
|
||||
stem=stem,
|
||||
remove_stop_words=remove_stop_words,
|
||||
custom_stop_words=custom_stop_words,
|
||||
ascii_folding=ascii_folding,
|
||||
ngram_min_length=ngram_min_length,
|
||||
ngram_max_length=ngram_max_length,
|
||||
prefix_only=prefix_only,
|
||||
block_size=block_size,
|
||||
)
|
||||
LOOP.run(
|
||||
self._table.create_index(
|
||||
|
||||
@@ -23,7 +23,7 @@ class AnswerdotaiRerankers(Reranker):
|
||||
column : str, default "text"
|
||||
The name of the column to use as input to the cross encoder model.
|
||||
return_score : str, default "relevance"
|
||||
options are "relevance" or "all". Only "relevance" is supported for now.
|
||||
options are "relevance" or "all".
|
||||
**kwargs
|
||||
Additional keyword arguments to pass to the model. For example, 'device'.
|
||||
See AnswerDotAI/rerankers for more information.
|
||||
@@ -77,12 +77,13 @@ class AnswerdotaiRerankers(Reranker):
|
||||
vector_results: pa.Table,
|
||||
fts_results: pa.Table,
|
||||
):
|
||||
combined_results = self.merge_results(vector_results, fts_results)
|
||||
if self.score == "all":
|
||||
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
|
||||
else:
|
||||
combined_results = self.merge_results(vector_results, fts_results)
|
||||
combined_results = self._rerank(combined_results, query)
|
||||
if self.score == "relevance":
|
||||
combined_results = self._keep_relevance_score(combined_results)
|
||||
elif self.score == "all":
|
||||
combined_results = self._merge_and_keep_scores(vector_results, fts_results)
|
||||
combined_results = combined_results.sort_by(
|
||||
[("_relevance_score", "descending")]
|
||||
)
|
||||
|
||||
@@ -16,7 +16,7 @@ class ColbertReranker(AnswerdotaiRerankers):
|
||||
column : str, default "text"
|
||||
The name of the column to use as input to the cross encoder model.
|
||||
return_score : str, default "relevance"
|
||||
options are "relevance" or "all". Only "relevance" is supported for now.
|
||||
options are "relevance" or "all".
|
||||
**kwargs
|
||||
Additional keyword arguments to pass to the model, for example, 'device'.
|
||||
See AnswerDotAI/rerankers for more information.
|
||||
|
||||
@@ -1102,10 +1102,12 @@ class Table(ABC):
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
wait_timeout: Optional[timedelta] = None,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
@@ -1168,6 +1170,9 @@ class Table(ABC):
|
||||
remove_stop_words : bool, default True
|
||||
Whether to remove stop words. Stop words are common words that are often
|
||||
removed from text before indexing. For example, in English "the" and "and".
|
||||
custom_stop_words : list of str, optional
|
||||
Custom words that replace the built-in language stop words. ``None``
|
||||
uses the built-in list; an empty list explicitly uses no stop words.
|
||||
ascii_folding : bool, default True
|
||||
Whether to fold ASCII characters. This converts accented characters to
|
||||
their ASCII equivalent. For example, "café" would be converted to "cafe".
|
||||
@@ -1177,6 +1182,10 @@ class Table(ABC):
|
||||
The maximum length of an n-gram.
|
||||
prefix_only: bool, default False
|
||||
Whether to only index the prefix of the token for ngram tokenizer.
|
||||
block_size: int, default 128
|
||||
The number of documents per compressed posting block. Must be 128
|
||||
or 256. A value of 256 uses the experimental FTS V3 format and
|
||||
may introduce breaking changes.
|
||||
wait_timeout: timedelta, optional
|
||||
The timeout to wait if indexing is asynchronous.
|
||||
name: str, optional
|
||||
@@ -3022,10 +3031,12 @@ class LanceTable(Table):
|
||||
lower_case: bool = True,
|
||||
stem: bool = True,
|
||||
remove_stop_words: bool = True,
|
||||
custom_stop_words: Optional[List[str]] = None,
|
||||
ascii_folding: bool = True,
|
||||
ngram_min_length: int = 3,
|
||||
ngram_max_length: int = 3,
|
||||
prefix_only: bool = False,
|
||||
block_size: int = 128,
|
||||
name: Optional[str] = None,
|
||||
):
|
||||
"""Create a full-text search index on a column.
|
||||
@@ -3067,6 +3078,7 @@ class LanceTable(Table):
|
||||
"lower_case": lower_case,
|
||||
"stem": stem,
|
||||
"remove_stop_words": remove_stop_words,
|
||||
"custom_stop_words": custom_stop_words,
|
||||
"ascii_folding": ascii_folding,
|
||||
"ngram_min_length": ngram_min_length,
|
||||
"ngram_max_length": ngram_max_length,
|
||||
@@ -3074,10 +3086,9 @@ class LanceTable(Table):
|
||||
}
|
||||
else:
|
||||
tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
|
||||
tokenizer_configs["custom_stop_words"] = custom_stop_words
|
||||
|
||||
config = FTS(
|
||||
**tokenizer_configs,
|
||||
)
|
||||
config = FTS(block_size=block_size, **tokenizer_configs)
|
||||
|
||||
try:
|
||||
LOOP.run(
|
||||
|
||||
@@ -219,11 +219,48 @@ def test_create_inverted_index(table, with_position):
|
||||
table.create_fts_index(
|
||||
"text",
|
||||
with_position=with_position,
|
||||
custom_stop_words=["puppy"],
|
||||
name="custom_fts_index",
|
||||
)
|
||||
indices = table.list_indices()
|
||||
fts_indices = [i for i in indices if i.index_type == "FTS"]
|
||||
assert any(i.name == "custom_fts_index" for i in fts_indices)
|
||||
assert fts_indices[0].index_details["custom_stop_words"] == ["puppy"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [128, 256])
|
||||
def test_create_inverted_index_block_size(table, block_size):
|
||||
table.create_index("text", config=FTS(block_size=block_size))
|
||||
|
||||
index = next(index for index in table.list_indices() if index.index_type == "FTS")
|
||||
assert index.index_details["block_size"] == block_size
|
||||
assert index.index_version == (2 if block_size == 128 else 3)
|
||||
|
||||
results = table.search("puppy").limit(5).to_list()
|
||||
assert len(results) == 5
|
||||
|
||||
|
||||
def test_create_inverted_index_rejects_invalid_block_size(table):
|
||||
with pytest.raises(ValueError, match="128 or 256"):
|
||||
table.create_index("text", config=FTS(block_size=129))
|
||||
|
||||
|
||||
def test_custom_stop_words_list(table):
|
||||
table.create_index(
|
||||
"text",
|
||||
config=FTS(stem=False, custom_stop_words=["lance"]),
|
||||
)
|
||||
|
||||
assert table.list_indices()[0].index_details["custom_stop_words"] == ["lance"]
|
||||
tokens = table.tokenize("the lance data", column="text")
|
||||
assert [token.text for token in tokens] == ["the", "data"]
|
||||
empty_tokens = ldb.tokenize("the lance data", stem=False, custom_stop_words=[])
|
||||
assert [token.text for token in empty_tokens] == ["the", "lance", "data"]
|
||||
with pytest.raises(TypeError, match=r"custom_stop_words.*int"):
|
||||
ldb.tokenize(
|
||||
"the lance data",
|
||||
custom_stop_words=["lance", 42],
|
||||
)
|
||||
|
||||
|
||||
def test_search_fts(table):
|
||||
|
||||
@@ -768,7 +768,11 @@ def test_table_create_indices():
|
||||
# Test create_fts_index with custom name (legacy method)
|
||||
with pytest.warns(DeprecationWarning, match="create_fts_index"):
|
||||
table.create_fts_index(
|
||||
"text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
|
||||
"text",
|
||||
wait_timeout=timedelta(seconds=2),
|
||||
block_size=256,
|
||||
custom_stop_words=["cloud"],
|
||||
name="custom_fts_idx",
|
||||
)
|
||||
|
||||
# Test create_index with custom name (legacy form: vector_column_name kwarg)
|
||||
@@ -791,6 +795,8 @@ def test_table_create_indices():
|
||||
fts_req = received_requests[1]
|
||||
assert "name" in fts_req
|
||||
assert fts_req["name"] == "custom_fts_idx"
|
||||
assert fts_req["block_size"] == 256
|
||||
assert fts_req["custom_stop_words"] == ["cloud"]
|
||||
|
||||
# Check vector index request has custom name
|
||||
vector_req = received_requests[2]
|
||||
@@ -876,7 +882,7 @@ def test_remote_create_index_new_api():
|
||||
_warnings.simplefilter("error", DeprecationWarning)
|
||||
table.create_index("vector", config=IvfPq(distance_type="l2"))
|
||||
table.create_index("category", config=BTree())
|
||||
table.create_index("text", config=FTS())
|
||||
table.create_index("text", config=FTS(block_size=256))
|
||||
# IvfRq via new API
|
||||
table.create_index("vector", config=IvfRq(distance_type="l2"))
|
||||
|
||||
@@ -896,6 +902,7 @@ def test_remote_create_index_new_api():
|
||||
"vector",
|
||||
"vector",
|
||||
]
|
||||
assert received_requests[2]["block_size"] == 256
|
||||
|
||||
|
||||
def test_table_wait_for_index_timeout():
|
||||
|
||||
@@ -644,6 +644,21 @@ def test_cross_encoder_reranker_return_all(tmp_path):
|
||||
assert "_distance" in result.column_names
|
||||
|
||||
|
||||
def test_answerdotai_reranker_return_all(tmp_path):
|
||||
pytest.importorskip("rerankers")
|
||||
reranker = AnswerdotaiRerankers(return_score="all")
|
||||
table, schema = get_test_table(tmp_path)
|
||||
query = "single player experience"
|
||||
result = (
|
||||
table.search(query, query_type="hybrid", vector_column_name="vector")
|
||||
.rerank(reranker=reranker)
|
||||
.to_arrow()
|
||||
)
|
||||
assert "_relevance_score" in result.column_names
|
||||
assert "_score" in result.column_names
|
||||
assert "_distance" in result.column_names
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regression tests for LinearCombinationReranker scoring bugs (issue #3154)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+7
-1
@@ -59,7 +59,11 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
|
||||
.ascii_folding(params.ascii_folding)
|
||||
.ngram_min_length(params.ngram_min_length)
|
||||
.ngram_max_length(params.ngram_max_length)
|
||||
.ngram_prefix_only(params.prefix_only);
|
||||
.ngram_prefix_only(params.prefix_only)
|
||||
.custom_stop_words(params.custom_stop_words);
|
||||
let inner_opts = inner_opts
|
||||
.block_size(params.block_size)
|
||||
.map_err(|err| PyValueError::new_err(err.to_string()))?;
|
||||
Ok(LanceDbIndex::FTS(inner_opts))
|
||||
}
|
||||
"IvfFlat" => {
|
||||
@@ -203,10 +207,12 @@ struct FtsParams {
|
||||
lower_case: bool,
|
||||
stem: bool,
|
||||
remove_stop_words: bool,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: bool,
|
||||
ngram_min_length: u32,
|
||||
ngram_max_length: u32,
|
||||
prefix_only: bool,
|
||||
block_size: usize,
|
||||
}
|
||||
|
||||
#[derive(FromPyObject)]
|
||||
|
||||
+4
-1
@@ -520,6 +520,7 @@ impl From<LanceDbFtsToken> for FtsToken {
|
||||
lower_case = true,
|
||||
stem = true,
|
||||
remove_stop_words = true,
|
||||
custom_stop_words = None,
|
||||
ascii_folding = true,
|
||||
ngram_min_length = 3,
|
||||
ngram_max_length = 3,
|
||||
@@ -534,6 +535,7 @@ pub fn tokenize(
|
||||
lower_case: bool,
|
||||
stem: bool,
|
||||
remove_stop_words: bool,
|
||||
custom_stop_words: Option<Vec<String>>,
|
||||
ascii_folding: bool,
|
||||
ngram_min_length: u32,
|
||||
ngram_max_length: u32,
|
||||
@@ -555,7 +557,8 @@ pub fn tokenize(
|
||||
.ascii_folding(ascii_folding)
|
||||
.ngram_min_length(ngram_min_length)
|
||||
.ngram_max_length(ngram_max_length)
|
||||
.ngram_prefix_only(prefix_only);
|
||||
.ngram_prefix_only(prefix_only)
|
||||
.custom_stop_words(custom_stop_words);
|
||||
let tokens = lancedb_tokenize(&query, ¶ms).infer_error()?;
|
||||
Ok(tokens.into_iter().map(FtsToken::from).collect())
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.32.0-beta.2"
|
||||
version = "0.32.0-beta.3"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
|
||||
@@ -76,7 +76,12 @@ async fn create_table(db: &Connection) -> Result<Table> {
|
||||
|
||||
async fn create_index(table: &Table) -> Result<()> {
|
||||
table
|
||||
.create_index(&["doc"], Index::FTS(FtsIndexBuilder::default()))
|
||||
.create_index(
|
||||
&["doc"],
|
||||
Index::FTS(
|
||||
FtsIndexBuilder::default().custom_stop_words(Some(vec!["example".to_owned()])),
|
||||
),
|
||||
)
|
||||
.execute()
|
||||
.await?;
|
||||
Ok(())
|
||||
|
||||
@@ -273,7 +273,11 @@ pub(crate) async fn take_blobs_aligned(
|
||||
if *is_null {
|
||||
builder.append_null();
|
||||
} else {
|
||||
builder.append_value(payloads[payload_idx].data.as_ref());
|
||||
if let Some(data) = &payloads[payload_idx].data {
|
||||
builder.append_value(data);
|
||||
} else {
|
||||
builder.append_null();
|
||||
}
|
||||
payload_idx += 1;
|
||||
}
|
||||
}
|
||||
@@ -315,7 +319,7 @@ pub(crate) async fn take_blob_files_aligned(
|
||||
if *is_null {
|
||||
None
|
||||
} else {
|
||||
Some(handles.next().unwrap())
|
||||
handles.next().flatten()
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
|
||||
@@ -54,7 +54,26 @@ pub enum Index {
|
||||
/// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index.
|
||||
Fm(FmIndexBuilder),
|
||||
|
||||
/// Full text search index using bm25.
|
||||
/// Full text search index using BM25.
|
||||
///
|
||||
/// The posting block size defaults to 128. Supported values are 128 and 256;
|
||||
/// a value of 256 uses the experimental FTS V3 format and may introduce
|
||||
/// breaking changes.
|
||||
///
|
||||
/// ```
|
||||
/// use lancedb::index::{Index, scalar::FtsIndexBuilder};
|
||||
///
|
||||
/// # async fn create_fts_index(
|
||||
/// # table: &lancedb::Table,
|
||||
/// # ) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let params = FtsIndexBuilder::default().block_size(256)?;
|
||||
/// table
|
||||
/// .create_index(&["text"], Index::FTS(params))
|
||||
/// .execute()
|
||||
/// .await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
FTS(FtsIndexBuilder),
|
||||
|
||||
/// IVF index
|
||||
|
||||
@@ -4496,6 +4496,28 @@ mod tests {
|
||||
serde_json::to_value(InvertedIndexParams::default()).unwrap(),
|
||||
Index::FTS(Default::default()),
|
||||
),
|
||||
(
|
||||
"FTS",
|
||||
{
|
||||
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
|
||||
body["block_size"] = 256.into();
|
||||
body
|
||||
},
|
||||
Index::FTS(InvertedIndexParams::default().block_size(256).unwrap()),
|
||||
),
|
||||
(
|
||||
"FTS",
|
||||
{
|
||||
let mut body = serde_json::to_value(InvertedIndexParams::default()).unwrap();
|
||||
body["custom_stop_words"] = json!(["cat", " cat ", "CAT"]);
|
||||
body
|
||||
},
|
||||
Index::FTS(InvertedIndexParams::default().custom_stop_words(Some(vec![
|
||||
"cat".to_string(),
|
||||
" cat ".to_string(),
|
||||
"CAT".to_string(),
|
||||
]))),
|
||||
),
|
||||
];
|
||||
|
||||
for (index_type, expected_body, index) in cases {
|
||||
@@ -5027,8 +5049,9 @@ mod tests {
|
||||
"max_token_length": 40,
|
||||
"lower_case": true,
|
||||
"stem": false,
|
||||
"remove_stop_words": false,
|
||||
"remove_stop_words": true,
|
||||
"ascii_folding": true,
|
||||
"custom_stop_words": ["hello"],
|
||||
})
|
||||
.to_string();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
@@ -5066,10 +5089,6 @@ mod tests {
|
||||
assert_eq!(
|
||||
tokens,
|
||||
vec![
|
||||
FtsToken {
|
||||
text: "hello".to_string(),
|
||||
position: 0,
|
||||
},
|
||||
FtsToken {
|
||||
text: "こんにちは".to_string(),
|
||||
position: 1,
|
||||
|
||||
@@ -21,6 +21,7 @@ use lance::dataset::WriteMode;
|
||||
use lance::dataset::builder::DatasetBuilder;
|
||||
use lance::dataset::{InsertBuilder, WriteParams};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance::index::scalar::load_segment_params;
|
||||
use lance::io::{ObjectStoreParams, WrappingObjectStore};
|
||||
use lance_datafusion::utils::StreamingWriteSource;
|
||||
use lance_index::IndexCriteria;
|
||||
@@ -3131,10 +3132,9 @@ impl BaseTable for NativeTable {
|
||||
async fn list_indices(&self) -> Result<Vec<IndexConfig>> {
|
||||
let dataset = self.dataset.get().await?;
|
||||
let total_rows = dataset.count_rows(None).await? as u64;
|
||||
let indices = dataset
|
||||
.describe_indices(None)
|
||||
.await?
|
||||
.into_iter()
|
||||
let descriptions = dataset.describe_indices(None).await?;
|
||||
let mut indices: Vec<IndexConfig> = descriptions
|
||||
.iter()
|
||||
.filter_map(|idx_desc| {
|
||||
let index_type: crate::index::IndexType = idx_desc
|
||||
.index_type()
|
||||
@@ -3192,6 +3192,31 @@ impl BaseTable for NativeTable {
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for index in indices
|
||||
.iter_mut()
|
||||
.filter(|index| index.index_type == crate::index::IndexType::FTS)
|
||||
{
|
||||
let Some(description) = descriptions
|
||||
.iter()
|
||||
.find(|description| description.name() == index.name)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let segments = description.segments();
|
||||
let Some(segment) = segments.first() else {
|
||||
continue;
|
||||
};
|
||||
let params = load_segment_params(&dataset, segment).await?;
|
||||
let details = serde_json::to_string(¶ms).map_err(|source| Error::Other {
|
||||
message: format!(
|
||||
"Failed to serialize full text search configuration for index '{}'",
|
||||
index.name
|
||||
),
|
||||
source: Some(Box::new(source)),
|
||||
})?;
|
||||
index.index_details = Some(details);
|
||||
}
|
||||
Ok(indices)
|
||||
}
|
||||
|
||||
|
||||
@@ -382,7 +382,9 @@ mod tests {
|
||||
use crate::connect;
|
||||
use crate::connection::ConnectBuilder;
|
||||
use crate::index::Index;
|
||||
use crate::index::scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder};
|
||||
use crate::index::scalar::{
|
||||
BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, FtsIndexBuilder,
|
||||
};
|
||||
use crate::index::vector::{
|
||||
IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder,
|
||||
};
|
||||
@@ -1362,16 +1364,57 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
table
|
||||
.create_index(&["text"], Index::FTS(Default::default()))
|
||||
.create_index(
|
||||
&["text"],
|
||||
Index::FTS(
|
||||
FtsIndexBuilder::default()
|
||||
.stem(false)
|
||||
.custom_stop_words(Some(vec!["cat".to_string()]))
|
||||
.block_size(256)
|
||||
.unwrap(),
|
||||
),
|
||||
)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
drop(table);
|
||||
let table = conn.open_table("test_bitmap").execute().await.unwrap();
|
||||
let index_configs = table.list_indices().await.unwrap();
|
||||
assert_eq!(index_configs.len(), 1);
|
||||
let index = index_configs.into_iter().next().unwrap();
|
||||
assert_eq!(index.index_type, crate::index::IndexType::FTS);
|
||||
assert_eq!(index.columns, vec!["text".to_string()]);
|
||||
assert_eq!(index.name, "text_idx");
|
||||
assert_eq!(index.index_version, Some(3));
|
||||
let index_params: FtsIndexBuilder =
|
||||
serde_json::from_str(index.index_details.as_deref().unwrap()).unwrap();
|
||||
assert_eq!(index_params.posting_block_size(), 256);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&index_params).unwrap()["custom_stop_words"],
|
||||
serde_json::json!(["cat"])
|
||||
);
|
||||
assert_eq!(
|
||||
table
|
||||
.tokenize("cat dog", "text_idx")
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|token| token.text)
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["dog"]
|
||||
);
|
||||
|
||||
let batches = table
|
||||
.query()
|
||||
.full_text_search(FullTextSearchQuery::new("cat dog".to_string()))
|
||||
.limit(120)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::<usize>(), 40);
|
||||
|
||||
let num_rows = 120;
|
||||
let stats = table.index_stats("text_idx").await.unwrap().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user