mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-05 03:12:57 +00:00
Compare commits
16 Commits
python-v0.
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d67a8743ba | ||
|
|
46fcbbc1e3 | ||
|
|
ff53b76ac0 | ||
|
|
2adb10e6a8 | ||
|
|
ac164c352b | ||
|
|
8bcac7e372 | ||
|
|
e496184ab2 | ||
|
|
d85d338a8e | ||
|
|
f0320725b6 | ||
|
|
dfbd3552bf | ||
|
|
1cf7b4b678 | ||
|
|
8ae4f42fbe | ||
|
|
0667fa38d4 | ||
|
|
30108c0b1f | ||
|
|
1628f7e3f3 | ||
|
|
2fd712312f |
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.23.0"
|
current_version = "0.23.1"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
101
Cargo.lock
generated
101
Cargo.lock
generated
@@ -3141,8 +3141,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fsst"
|
name = "fsst"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5ffdff7a2d68d22afc0657eddde3e946371ce7cfe730a3f78a5ed44ea5b1cb2e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"rand 0.9.2",
|
"rand 0.9.2",
|
||||||
@@ -4261,7 +4262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
|
checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"equivalent",
|
"equivalent",
|
||||||
"hashbrown 0.15.5",
|
"hashbrown 0.16.0",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_core",
|
"serde_core",
|
||||||
]
|
]
|
||||||
@@ -4478,8 +4479,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance"
|
name = "lance"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8c439decbc304e180748e34bb6d3df729069a222e83e74e2185c38f107136e9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4544,8 +4546,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-arrow"
|
name = "lance-arrow"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f4ee5508b225456d3d56998eaeef0d8fbce5ea93856df47b12a94d2e74153210"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4563,8 +4566,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-bitpacking"
|
name = "lance-bitpacking"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d1c065fb3bd4a8cc4f78428443e990d4921aa08f707b676753db740e0b402a21"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrayref",
|
"arrayref",
|
||||||
"paste",
|
"paste",
|
||||||
@@ -4573,8 +4577,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-core"
|
name = "lance-core"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8856abad92e624b75cd57a04703f6441948a239463bdf973f2ac1924b0bcdbe"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4610,8 +4615,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datafusion"
|
name = "lance-datafusion"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4c8835308044cef5467d7751be87fcbefc2db01c22370726a8704bd62991693f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4641,8 +4647,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datagen"
|
name = "lance-datagen"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "612de1e888bb36f6bf51196a6eb9574587fdf256b1759a4c50e643e00d5f96d0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4659,8 +4666,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-encoding"
|
name = "lance-encoding"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2b456b29b135d3c7192602e516ccade38b5483986e121895fa43cf1fdb38bf60"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4697,8 +4705,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-file"
|
name = "lance-file"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ab1538d14d5bb3735b4222b3f5aff83cfa59cc6ef7cdd3dd9139e4c77193c80b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4730,8 +4739,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-geo"
|
name = "lance-geo"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a5a69a2f3b55703d9c240ad7c5ffa2c755db69e9cf8aa05efe274a212910472d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"datafusion",
|
"datafusion",
|
||||||
"geo-types",
|
"geo-types",
|
||||||
@@ -4742,8 +4752,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-index"
|
name = "lance-index"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0ea84613df6fa6b9168a1f056ba4f9cb73b90a1b452814c6fd4b3529bcdbfc78"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4804,8 +4815,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-io"
|
name = "lance-io"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6b3fc4c1d941fceef40a0edbd664dbef108acfc5d559bb9e7f588d0c733cbc35"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4845,8 +4857,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-linalg"
|
name = "lance-linalg"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b62ffbc5ce367fbf700a69de3fe0612ee1a11191a64a632888610b6bacfa0f63"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4862,8 +4875,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace"
|
name = "lance-namespace"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "791bbcd868ee758123a34e07d320a1fb99379432b5ecc0e78d6b4686e999b629"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -4875,8 +4889,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace-impls"
|
name = "lance-namespace-impls"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ee713505576f6b1988a491f77c7ca8b0cf7090a393598e63c85079fa70a53ebf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-ipc",
|
"arrow-ipc",
|
||||||
@@ -4918,8 +4933,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-table"
|
name = "lance-table"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6fdb2d56bfa4d1511c765fa0cc00fdaa37e5d2d1cd2f57b3c6355d9072177052"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4958,8 +4974,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-testing"
|
name = "lance-testing"
|
||||||
version = "1.0.1-beta.1"
|
version = "1.0.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d8ccb1a4a9284435c6a8c02c8c06e7e041bece0d7f722152159353cf55dc51e3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-schema",
|
"arrow-schema",
|
||||||
@@ -4970,7 +4987,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.23.0"
|
version = "0.23.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash",
|
"ahash",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
@@ -5049,7 +5066,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
version = "0.23.0"
|
version = "0.23.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-ipc",
|
"arrow-ipc",
|
||||||
@@ -5069,7 +5086,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.26.0"
|
version = "0.26.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -6725,8 +6742,8 @@ version = "0.13.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.4.1",
|
"heck 0.5.0",
|
||||||
"itertools 0.12.1",
|
"itertools 0.14.0",
|
||||||
"log",
|
"log",
|
||||||
"multimap",
|
"multimap",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
@@ -6746,7 +6763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"itertools 0.12.1",
|
"itertools 0.14.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.106",
|
"syn 2.0.106",
|
||||||
@@ -8076,7 +8093,7 @@ version = "0.8.9"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"heck 0.4.1",
|
"heck 0.5.0",
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
"syn 2.0.106",
|
"syn 2.0.106",
|
||||||
|
|||||||
28
Cargo.toml
28
Cargo.toml
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
|
|||||||
rust-version = "1.78.0"
|
rust-version = "1.78.0"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
lance = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance = { "version" = "=1.0.1", default-features = false }
|
||||||
lance-core = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-core = "=1.0.1"
|
||||||
lance-datagen = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datagen = "=1.0.1"
|
||||||
lance-file = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-file = "=1.0.1"
|
||||||
lance-io = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-io = { "version" = "=1.0.1", default-features = false }
|
||||||
lance-index = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-index = "=1.0.1"
|
||||||
lance-linalg = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-linalg = "=1.0.1"
|
||||||
lance-namespace = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace = "=1.0.1"
|
||||||
lance-namespace-impls = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace-impls = { "version" = "=1.0.1", default-features = false }
|
||||||
lance-table = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-table = "=1.0.1"
|
||||||
lance-testing = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-testing = "=1.0.1"
|
||||||
lance-datafusion = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datafusion = "=1.0.1"
|
||||||
lance-encoding = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-encoding = "=1.0.1"
|
||||||
lance-arrow = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
lance-arrow = "=1.0.1"
|
||||||
ahash = "0.8"
|
ahash = "0.8"
|
||||||
# Note that this one does not include pyarrow
|
# Note that this one does not include pyarrow
|
||||||
arrow = { version = "56.2", optional = false }
|
arrow = { version = "56.2", optional = false }
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ check_command_exists() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if [[ ! -e ./lancedb ]]; then
|
if [[ ! -e ./lancedb ]]; then
|
||||||
if [[ -v SOPHON_READ_TOKEN ]]; then
|
if [[ x${SOPHON_READ_TOKEN} != "x" ]]; then
|
||||||
INPUT="lancedb-linux-x64"
|
INPUT="lancedb-linux-x64"
|
||||||
gh release \
|
gh release \
|
||||||
--repo lancedb/lancedb \
|
--repo lancedb/lancedb \
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ watch:
|
|||||||
theme:
|
theme:
|
||||||
name: "material"
|
name: "material"
|
||||||
logo: assets/logo.png
|
logo: assets/logo.png
|
||||||
favicon: assets/logo.png
|
favicon: assets/favicon.ico
|
||||||
palette:
|
palette:
|
||||||
# Palette toggle for light mode
|
# Palette toggle for light mode
|
||||||
- scheme: lancedb
|
- scheme: lancedb
|
||||||
@@ -32,8 +32,6 @@ theme:
|
|||||||
- content.tooltips
|
- content.tooltips
|
||||||
- toc.follow
|
- toc.follow
|
||||||
- navigation.top
|
- navigation.top
|
||||||
- navigation.tabs
|
|
||||||
- navigation.tabs.sticky
|
|
||||||
- navigation.footer
|
- navigation.footer
|
||||||
- navigation.tracking
|
- navigation.tracking
|
||||||
- navigation.instant
|
- navigation.instant
|
||||||
@@ -115,12 +113,13 @@ markdown_extensions:
|
|||||||
emoji_index: !!python/name:material.extensions.emoji.twemoji
|
emoji_index: !!python/name:material.extensions.emoji.twemoji
|
||||||
emoji_generator: !!python/name:material.extensions.emoji.to_svg
|
emoji_generator: !!python/name:material.extensions.emoji.to_svg
|
||||||
- markdown.extensions.toc:
|
- markdown.extensions.toc:
|
||||||
baselevel: 1
|
toc_depth: 3
|
||||||
permalink: ""
|
permalink: true
|
||||||
|
permalink_title: Anchor link to this section
|
||||||
|
|
||||||
nav:
|
nav:
|
||||||
- API reference:
|
- Documentation:
|
||||||
- Overview: index.md
|
- SDK Reference: index.md
|
||||||
- Python: python/python.md
|
- Python: python/python.md
|
||||||
- Javascript/TypeScript: js/globals.md
|
- Javascript/TypeScript: js/globals.md
|
||||||
- Java: java/java.md
|
- Java: java/java.md
|
||||||
|
|||||||
BIN
docs/src/assets/favicon.ico
Normal file
BIN
docs/src/assets/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
@@ -0,0 +1,111 @@
|
|||||||
|
# VoyageAI Embeddings : Multimodal
|
||||||
|
|
||||||
|
VoyageAI embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
|
||||||
|
under [https://docs.voyageai.com/docs/multimodal-embeddings](https://docs.voyageai.com/docs/multimodal-embeddings)
|
||||||
|
|
||||||
|
Supported multimodal models:
|
||||||
|
|
||||||
|
- `voyage-multimodal-3` - 1024 dimensions (text + images)
|
||||||
|
- `voyage-multimodal-3.5` - Flexible dimensions (256, 512, 1024 default, 2048). Supports text, images, and video.
|
||||||
|
|
||||||
|
### Video Support (voyage-multimodal-3.5)
|
||||||
|
|
||||||
|
The `voyage-multimodal-3.5` model supports video input through:
|
||||||
|
- Video URLs (`.mp4`, `.webm`, `.mov`, `.avi`, `.mkv`, `.m4v`, `.gif`)
|
||||||
|
- Video file paths
|
||||||
|
|
||||||
|
Constraints: Max 20MB video size.
|
||||||
|
|
||||||
|
Supported parameters (to be passed in `create` method) are:
|
||||||
|
|
||||||
|
| Parameter | Type | Default Value | Description |
|
||||||
|
|---|---|-------------------------|-------------------------------------------|
|
||||||
|
| `name` | `str` | `"voyage-multimodal-3"` | The model ID of the VoyageAI model to use |
|
||||||
|
| `output_dimension` | `int` | `None` | Output dimension for voyage-multimodal-3.5. Valid: 256, 512, 1024, 2048 |
|
||||||
|
|
||||||
|
Usage Example:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import lancedb
|
||||||
|
from lancedb.pydantic import LanceModel, Vector
|
||||||
|
from lancedb.embeddings import get_registry
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
os.environ['VOYAGE_API_KEY'] = 'YOUR_VOYAGE_API_KEY'
|
||||||
|
|
||||||
|
db = lancedb.connect(".lancedb")
|
||||||
|
func = get_registry().get("voyageai").create(name="voyage-multimodal-3")
|
||||||
|
|
||||||
|
|
||||||
|
def image_to_base64(image_bytes: bytes):
|
||||||
|
buffered = BytesIO(image_bytes)
|
||||||
|
img_str = base64.b64encode(buffered.getvalue())
|
||||||
|
return img_str.decode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
class Images(LanceModel):
|
||||||
|
label: str
|
||||||
|
image_uri: str = func.SourceField() # image uri as the source
|
||||||
|
image_bytes: str = func.SourceField() # image bytes base64 encoded as the source
|
||||||
|
vector: Vector(func.ndims()) = func.VectorField() # vector column
|
||||||
|
vec_from_bytes: Vector(func.ndims()) = func.VectorField() # Another vector column
|
||||||
|
|
||||||
|
|
||||||
|
if "images" in db.table_names():
|
||||||
|
db.drop_table("images")
|
||||||
|
table = db.create_table("images", schema=Images)
|
||||||
|
labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
|
||||||
|
uris = [
|
||||||
|
"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
|
||||||
|
"http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
|
||||||
|
"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
|
||||||
|
"http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
|
||||||
|
"http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
|
||||||
|
"http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
|
||||||
|
]
|
||||||
|
# get each uri as bytes
|
||||||
|
images_bytes = [image_to_base64(requests.get(uri).content) for uri in uris]
|
||||||
|
table.add(
|
||||||
|
pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": images_bytes})
|
||||||
|
)
|
||||||
|
```
|
||||||
|
Now we can search using text from both the default vector column and the custom vector column
|
||||||
|
```python
|
||||||
|
|
||||||
|
# text search
|
||||||
|
actual = table.search("man's best friend", "vec_from_bytes").limit(1).to_pydantic(Images)[0]
|
||||||
|
print(actual.label) # prints "dog"
|
||||||
|
|
||||||
|
frombytes = (
|
||||||
|
table.search("man's best friend", vector_column_name="vec_from_bytes")
|
||||||
|
.limit(1)
|
||||||
|
.to_pydantic(Images)[0]
|
||||||
|
)
|
||||||
|
print(frombytes.label)
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
Because we're using a multi-modal embedding function, we can also search using images
|
||||||
|
|
||||||
|
```python
|
||||||
|
# image search
|
||||||
|
query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
|
||||||
|
image_bytes = requests.get(query_image_uri).content
|
||||||
|
query_image = Image.open(BytesIO(image_bytes))
|
||||||
|
actual = table.search(query_image, "vec_from_bytes").limit(1).to_pydantic(Images)[0]
|
||||||
|
print(actual.label == "dog")
|
||||||
|
|
||||||
|
# image search using a custom vector column
|
||||||
|
other = (
|
||||||
|
table.search(query_image, vector_column_name="vec_from_bytes")
|
||||||
|
.limit(1)
|
||||||
|
.to_pydantic(Images)[0]
|
||||||
|
)
|
||||||
|
print(actual.label)
|
||||||
|
|
||||||
|
```
|
||||||
@@ -1,8 +1,12 @@
|
|||||||
# API Reference
|
# SDK Reference
|
||||||
|
|
||||||
This page contains the API reference for the SDKs supported by the LanceDB team.
|
This site contains the API reference for the client SDKs supported by [LanceDB](https://lancedb.com).
|
||||||
|
|
||||||
- [Python](python/python.md)
|
- [Python](python/python.md)
|
||||||
- [JavaScript/TypeScript](js/globals.md)
|
- [JavaScript/TypeScript](js/globals.md)
|
||||||
- [Java](java/java.md)
|
- [Java](java/java.md)
|
||||||
- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)
|
- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)
|
||||||
|
|
||||||
|
!!! info "LanceDB Documentation"
|
||||||
|
|
||||||
|
If you're looking for the full documentation of LanceDB, visit [docs.lancedb.com](https://docs.lancedb.com).
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-core</artifactId>
|
<artifactId>lancedb-core</artifactId>
|
||||||
<version>0.23.0</version>
|
<version>0.23.1</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -85,17 +85,26 @@
|
|||||||
|
|
||||||
/* Header gradient (only header area) */
|
/* Header gradient (only header area) */
|
||||||
.md-header {
|
.md-header {
|
||||||
background: linear-gradient(90deg, #3B2E58 0%, #F0B7C1 45%, #E55A2B 100%);
|
background: linear-gradient(90deg, #e4d8f8 0%, #F0B7C1 45%, #E55A2B 100%);
|
||||||
box-shadow: inset 0 1px 0 rgba(255,255,255,0.08), 0 1px 0 rgba(0,0,0,0.08);
|
box-shadow: inset 0 1px 0 rgba(255,255,255,0.08), 0 1px 0 rgba(0,0,0,0.08);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Improve brand title contrast on the lavender side */
|
||||||
|
.md-header__title,
|
||||||
|
.md-header__topic,
|
||||||
|
.md-header__title .md-ellipsis,
|
||||||
|
.md-header__topic .md-ellipsis {
|
||||||
|
color: #2b1b3a;
|
||||||
|
text-shadow: 0 1px 0 rgba(255, 255, 255, 0.25);
|
||||||
|
}
|
||||||
|
|
||||||
/* Same colors as header for tabs (that hold the text) */
|
/* Same colors as header for tabs (that hold the text) */
|
||||||
.md-tabs {
|
.md-tabs {
|
||||||
background: linear-gradient(90deg, #3B2E58 0%, #F0B7C1 45%, #E55A2B 100%);
|
background: linear-gradient(90deg, #e4d8f8 0%, #F0B7C1 45%, #E55A2B 100%);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Dark scheme variant */
|
/* Dark scheme variant */
|
||||||
[data-md-color-scheme="slate"] .md-header,
|
[data-md-color-scheme="slate"] .md-header,
|
||||||
[data-md-color-scheme="slate"] .md-tabs {
|
[data-md-color-scheme="slate"] .md-tabs {
|
||||||
background: linear-gradient(90deg, #3B2E58 0%, #F0B7C1 45%, #E55A2B 100%);
|
background: linear-gradient(90deg, #e4d8f8 0%, #F0B7C1 45%, #E55A2B 100%);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.23.0-final.0</version>
|
<version>0.23.1-final.0</version>
|
||||||
<relativePath>../pom.xml</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.23.0-final.0</version>
|
<version>0.23.1-final.0</version>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<name>${project.artifactId}</name>
|
<name>${project.artifactId}</name>
|
||||||
<description>LanceDB Java SDK Parent POM</description>
|
<description>LanceDB Java SDK Parent POM</description>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
version = "0.23.0"
|
version = "0.23.1"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description.workspace = true
|
description.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-arm64",
|
"name": "@lancedb/lancedb-darwin-arm64",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.darwin-arm64.node",
|
"main": "lancedb.darwin-arm64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-x64",
|
"name": "@lancedb/lancedb-darwin-x64",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.darwin-x64.node",
|
"main": "lancedb.darwin-x64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-gnu.node",
|
"main": "lancedb.linux-arm64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-musl.node",
|
"main": "lancedb.linux-arm64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-gnu.node",
|
"main": "lancedb.linux-x64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-musl.node",
|
"main": "lancedb.linux-x64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": [
|
"os": [
|
||||||
"win32"
|
"win32"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"os": ["win32"],
|
"os": ["win32"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.win32-x64-msvc.node",
|
"main": "lancedb.win32-x64-msvc.node",
|
||||||
|
|||||||
4
nodejs/package-lock.json
generated
4
nodejs/package-lock.json
generated
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64",
|
"x64",
|
||||||
"arm64"
|
"arm64"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
"ann"
|
"ann"
|
||||||
],
|
],
|
||||||
"private": false,
|
"private": false,
|
||||||
"version": "0.23.0",
|
"version": "0.23.1",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./dist/index.js",
|
".": "./dist/index.js",
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.26.1-beta.0"
|
current_version = "0.26.1"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.26.1-beta.0"
|
version = "0.26.1"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ __version__ = importlib.metadata.version("lancedb")
|
|||||||
|
|
||||||
from ._lancedb import connect as lancedb_connect
|
from ._lancedb import connect as lancedb_connect
|
||||||
from .common import URI, sanitize_uri
|
from .common import URI, sanitize_uri
|
||||||
|
from urllib.parse import urlparse
|
||||||
from .db import AsyncConnection, DBConnection, LanceDBConnection
|
from .db import AsyncConnection, DBConnection, LanceDBConnection
|
||||||
from .io import StorageOptionsProvider
|
from .io import StorageOptionsProvider
|
||||||
from .remote import ClientConfig
|
from .remote import ClientConfig
|
||||||
@@ -28,6 +29,39 @@ from .namespace import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_s3_bucket_with_dots(
|
||||||
|
uri: str, storage_options: Optional[Dict[str, str]]
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Check if an S3 URI has a bucket name containing dots and warn if no region
|
||||||
|
is specified. S3 buckets with dots cannot use virtual-hosted-style URLs,
|
||||||
|
which breaks automatic region detection.
|
||||||
|
|
||||||
|
See: https://github.com/lancedb/lancedb/issues/1898
|
||||||
|
"""
|
||||||
|
if not isinstance(uri, str) or not uri.startswith("s3://"):
|
||||||
|
return
|
||||||
|
|
||||||
|
parsed = urlparse(uri)
|
||||||
|
bucket = parsed.netloc
|
||||||
|
|
||||||
|
if "." not in bucket:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Check if region is provided in storage_options
|
||||||
|
region_keys = {"region", "aws_region"}
|
||||||
|
has_region = storage_options and any(k in storage_options for k in region_keys)
|
||||||
|
|
||||||
|
if not has_region:
|
||||||
|
raise ValueError(
|
||||||
|
f"S3 bucket name '{bucket}' contains dots, which prevents automatic "
|
||||||
|
f"region detection. Please specify the region explicitly via "
|
||||||
|
f"storage_options={{'region': '<your-region>'}} or "
|
||||||
|
f"storage_options={{'aws_region': '<your-region>'}}. "
|
||||||
|
f"See https://github.com/lancedb/lancedb/issues/1898 for details."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def connect(
|
def connect(
|
||||||
uri: URI,
|
uri: URI,
|
||||||
*,
|
*,
|
||||||
@@ -121,9 +155,11 @@ def connect(
|
|||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
_check_s3_bucket_with_dots(str(uri), storage_options)
|
||||||
|
|
||||||
if kwargs:
|
if kwargs:
|
||||||
raise ValueError(f"Unknown keyword arguments: {kwargs}")
|
raise ValueError(f"Unknown keyword arguments: {kwargs}")
|
||||||
|
|
||||||
return LanceDBConnection(
|
return LanceDBConnection(
|
||||||
uri,
|
uri,
|
||||||
read_consistency_interval=read_consistency_interval,
|
read_consistency_interval=read_consistency_interval,
|
||||||
@@ -211,6 +247,8 @@ async def connect_async(
|
|||||||
if isinstance(client_config, dict):
|
if isinstance(client_config, dict):
|
||||||
client_config = ClientConfig(**client_config)
|
client_config = ClientConfig(**client_config)
|
||||||
|
|
||||||
|
_check_s3_bucket_with_dots(str(uri), storage_options)
|
||||||
|
|
||||||
return AsyncConnection(
|
return AsyncConnection(
|
||||||
await lancedb_connect(
|
await lancedb_connect(
|
||||||
sanitize_uri(uri),
|
sanitize_uri(uri),
|
||||||
|
|||||||
@@ -210,10 +210,8 @@ class DBConnection(EnforceOverrides):
|
|||||||
page_token: str, optional
|
page_token: str, optional
|
||||||
The token to use for pagination. If not present, start from the beginning.
|
The token to use for pagination. If not present, start from the beginning.
|
||||||
Typically, this token is last table name from the previous page.
|
Typically, this token is last table name from the previous page.
|
||||||
Only supported by LanceDb Cloud.
|
|
||||||
limit: int, default 10
|
limit: int, default 10
|
||||||
The size of the page to return.
|
The size of the page to return.
|
||||||
Only supported by LanceDb Cloud.
|
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
import base64
|
import base64
|
||||||
import os
|
import os
|
||||||
from typing import ClassVar, TYPE_CHECKING, List, Union, Any, Generator
|
from typing import ClassVar, TYPE_CHECKING, List, Union, Any, Generator, Optional
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
@@ -45,11 +45,29 @@ def is_valid_url(text):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
VIDEO_EXTENSIONS = {".mp4", ".webm", ".mov", ".avi", ".mkv", ".m4v", ".gif"}
|
||||||
|
|
||||||
|
|
||||||
|
def is_video_url(url: str) -> bool:
|
||||||
|
"""Check if URL points to a video file based on extension."""
|
||||||
|
parsed = urlparse(url)
|
||||||
|
path = parsed.path.lower()
|
||||||
|
return any(path.endswith(ext) for ext in VIDEO_EXTENSIONS)
|
||||||
|
|
||||||
|
|
||||||
|
def is_video_path(path: Path) -> bool:
|
||||||
|
"""Check if file path is a video file based on extension."""
|
||||||
|
return path.suffix.lower() in VIDEO_EXTENSIONS
|
||||||
|
|
||||||
|
|
||||||
def transform_input(input_data: Union[str, bytes, Path]):
|
def transform_input(input_data: Union[str, bytes, Path]):
|
||||||
PIL = attempt_import_or_raise("PIL", "pillow")
|
PIL = attempt_import_or_raise("PIL", "pillow")
|
||||||
if isinstance(input_data, str):
|
if isinstance(input_data, str):
|
||||||
if is_valid_url(input_data):
|
if is_valid_url(input_data):
|
||||||
content = {"type": "image_url", "image_url": input_data}
|
if is_video_url(input_data):
|
||||||
|
content = {"type": "video_url", "video_url": input_data}
|
||||||
|
else:
|
||||||
|
content = {"type": "image_url", "image_url": input_data}
|
||||||
else:
|
else:
|
||||||
content = {"type": "text", "text": input_data}
|
content = {"type": "text", "text": input_data}
|
||||||
elif isinstance(input_data, PIL.Image.Image):
|
elif isinstance(input_data, PIL.Image.Image):
|
||||||
@@ -70,14 +88,24 @@ def transform_input(input_data: Union[str, bytes, Path]):
|
|||||||
"image_base64": "data:image/jpeg;base64," + img_str,
|
"image_base64": "data:image/jpeg;base64," + img_str,
|
||||||
}
|
}
|
||||||
elif isinstance(input_data, Path):
|
elif isinstance(input_data, Path):
|
||||||
img = PIL.Image.open(input_data)
|
if is_video_path(input_data):
|
||||||
buffered = BytesIO()
|
# Read video file and encode as base64
|
||||||
img.save(buffered, format="JPEG")
|
with open(input_data, "rb") as f:
|
||||||
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
video_bytes = f.read()
|
||||||
content = {
|
video_str = base64.b64encode(video_bytes).decode("utf-8")
|
||||||
"type": "image_base64",
|
content = {
|
||||||
"image_base64": "data:image/jpeg;base64," + img_str,
|
"type": "video_base64",
|
||||||
}
|
"video_base64": video_str,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
img = PIL.Image.open(input_data)
|
||||||
|
buffered = BytesIO()
|
||||||
|
img.save(buffered, format="JPEG")
|
||||||
|
img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
|
||||||
|
content = {
|
||||||
|
"type": "image_base64",
|
||||||
|
"image_base64": "data:image/jpeg;base64," + img_str,
|
||||||
|
}
|
||||||
else:
|
else:
|
||||||
raise ValueError("Each input should be either str, bytes, Path or Image.")
|
raise ValueError("Each input should be either str, bytes, Path or Image.")
|
||||||
|
|
||||||
@@ -91,6 +119,8 @@ def sanitize_multimodal_input(inputs: Union[TEXT, IMAGES]) -> List[Any]:
|
|||||||
PIL = attempt_import_or_raise("PIL", "pillow")
|
PIL = attempt_import_or_raise("PIL", "pillow")
|
||||||
if isinstance(inputs, (str, bytes, Path, PIL.Image.Image)):
|
if isinstance(inputs, (str, bytes, Path, PIL.Image.Image)):
|
||||||
inputs = [inputs]
|
inputs = [inputs]
|
||||||
|
elif isinstance(inputs, list):
|
||||||
|
pass # Already a list, use as-is
|
||||||
elif isinstance(inputs, pa.Array):
|
elif isinstance(inputs, pa.Array):
|
||||||
inputs = inputs.to_pylist()
|
inputs = inputs.to_pylist()
|
||||||
elif isinstance(inputs, pa.ChunkedArray):
|
elif isinstance(inputs, pa.ChunkedArray):
|
||||||
@@ -143,11 +173,16 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
* voyage-3
|
* voyage-3
|
||||||
* voyage-3-lite
|
* voyage-3-lite
|
||||||
* voyage-multimodal-3
|
* voyage-multimodal-3
|
||||||
|
* voyage-multimodal-3.5
|
||||||
* voyage-finance-2
|
* voyage-finance-2
|
||||||
* voyage-multilingual-2
|
* voyage-multilingual-2
|
||||||
* voyage-law-2
|
* voyage-law-2
|
||||||
* voyage-code-2
|
* voyage-code-2
|
||||||
|
|
||||||
|
output_dimension: int, optional
|
||||||
|
The output dimension for models that support flexible dimensions.
|
||||||
|
Currently only voyage-multimodal-3.5 supports this feature.
|
||||||
|
Valid options: 256, 512, 1024 (default), 2048.
|
||||||
|
|
||||||
Examples
|
Examples
|
||||||
--------
|
--------
|
||||||
@@ -175,7 +210,10 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
name: str
|
name: str
|
||||||
|
output_dimension: Optional[int] = None
|
||||||
client: ClassVar = None
|
client: ClassVar = None
|
||||||
|
_FLEXIBLE_DIM_MODELS: ClassVar[list] = ["voyage-multimodal-3.5"]
|
||||||
|
_VALID_DIMENSIONS: ClassVar[list] = [256, 512, 1024, 2048]
|
||||||
text_embedding_models: list = [
|
text_embedding_models: list = [
|
||||||
"voyage-3.5",
|
"voyage-3.5",
|
||||||
"voyage-3.5-lite",
|
"voyage-3.5-lite",
|
||||||
@@ -186,7 +224,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
"voyage-law-2",
|
"voyage-law-2",
|
||||||
"voyage-code-2",
|
"voyage-code-2",
|
||||||
]
|
]
|
||||||
multimodal_embedding_models: list = ["voyage-multimodal-3"]
|
multimodal_embedding_models: list = ["voyage-multimodal-3", "voyage-multimodal-3.5"]
|
||||||
contextual_embedding_models: list = ["voyage-context-3"]
|
contextual_embedding_models: list = ["voyage-context-3"]
|
||||||
|
|
||||||
def _is_multimodal_model(self, model_name: str):
|
def _is_multimodal_model(self, model_name: str):
|
||||||
@@ -198,6 +236,17 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
return model_name in self.contextual_embedding_models or "context" in model_name
|
return model_name in self.contextual_embedding_models or "context" in model_name
|
||||||
|
|
||||||
def ndims(self):
|
def ndims(self):
|
||||||
|
# Handle flexible dimension models
|
||||||
|
if self.name in self._FLEXIBLE_DIM_MODELS:
|
||||||
|
if self.output_dimension is not None:
|
||||||
|
if self.output_dimension not in self._VALID_DIMENSIONS:
|
||||||
|
raise ValueError(
|
||||||
|
f"Invalid output_dimension {self.output_dimension} "
|
||||||
|
f"for {self.name}. Valid options: {self._VALID_DIMENSIONS}"
|
||||||
|
)
|
||||||
|
return self.output_dimension
|
||||||
|
return 1024 # default dimension
|
||||||
|
|
||||||
if self.name == "voyage-3-lite":
|
if self.name == "voyage-3-lite":
|
||||||
return 512
|
return 512
|
||||||
elif self.name == "voyage-code-2":
|
elif self.name == "voyage-code-2":
|
||||||
@@ -211,12 +260,17 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
"voyage-finance-2",
|
"voyage-finance-2",
|
||||||
"voyage-multilingual-2",
|
"voyage-multilingual-2",
|
||||||
"voyage-law-2",
|
"voyage-law-2",
|
||||||
"voyage-multimodal-3",
|
|
||||||
]:
|
]:
|
||||||
return 1024
|
return 1024
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"Model {self.name} not supported")
|
raise ValueError(f"Model {self.name} not supported")
|
||||||
|
|
||||||
|
def _get_multimodal_kwargs(self, **kwargs):
|
||||||
|
"""Get kwargs for multimodal embed call, including output_dimension if set."""
|
||||||
|
if self.name in self._FLEXIBLE_DIM_MODELS and self.output_dimension is not None:
|
||||||
|
kwargs["output_dimension"] = self.output_dimension
|
||||||
|
return kwargs
|
||||||
|
|
||||||
def compute_query_embeddings(
|
def compute_query_embeddings(
|
||||||
self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
|
self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
|
||||||
) -> List[np.ndarray]:
|
) -> List[np.ndarray]:
|
||||||
@@ -234,6 +288,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
"""
|
"""
|
||||||
client = VoyageAIEmbeddingFunction._get_client()
|
client = VoyageAIEmbeddingFunction._get_client()
|
||||||
if self._is_multimodal_model(self.name):
|
if self._is_multimodal_model(self.name):
|
||||||
|
kwargs = self._get_multimodal_kwargs(**kwargs)
|
||||||
result = client.multimodal_embed(
|
result = client.multimodal_embed(
|
||||||
inputs=[[query]], model=self.name, input_type="query", **kwargs
|
inputs=[[query]], model=self.name, input_type="query", **kwargs
|
||||||
)
|
)
|
||||||
@@ -275,6 +330,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
)
|
)
|
||||||
if has_images:
|
if has_images:
|
||||||
# Use non-batched API for images
|
# Use non-batched API for images
|
||||||
|
kwargs = self._get_multimodal_kwargs(**kwargs)
|
||||||
result = client.multimodal_embed(
|
result = client.multimodal_embed(
|
||||||
inputs=sanitized, model=self.name, input_type="document", **kwargs
|
inputs=sanitized, model=self.name, input_type="document", **kwargs
|
||||||
)
|
)
|
||||||
@@ -357,6 +413,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
callable: A function that takes a batch of texts and returns embeddings.
|
callable: A function that takes a batch of texts and returns embeddings.
|
||||||
"""
|
"""
|
||||||
if self._is_multimodal_model(self.name):
|
if self._is_multimodal_model(self.name):
|
||||||
|
multimodal_kwargs = self._get_multimodal_kwargs(**kwargs)
|
||||||
|
|
||||||
def embed_batch(batch: List[str]) -> List[np.array]:
|
def embed_batch(batch: List[str]) -> List[np.array]:
|
||||||
batch_inputs = sanitize_multimodal_input(batch)
|
batch_inputs = sanitize_multimodal_input(batch)
|
||||||
@@ -364,7 +421,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
|
|||||||
inputs=batch_inputs,
|
inputs=batch_inputs,
|
||||||
model=self.name,
|
model=self.name,
|
||||||
input_type=input_type,
|
input_type=input_type,
|
||||||
**kwargs,
|
**multimodal_kwargs,
|
||||||
)
|
)
|
||||||
return result.embeddings
|
return result.embeddings
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,17 @@ from lancedb._lancedb import (
|
|||||||
UpdateResult,
|
UpdateResult,
|
||||||
)
|
)
|
||||||
from lancedb.embeddings.base import EmbeddingFunctionConfig
|
from lancedb.embeddings.base import EmbeddingFunctionConfig
|
||||||
from lancedb.index import FTS, BTree, Bitmap, HnswSq, IvfFlat, IvfPq, IvfSq, LabelList
|
from lancedb.index import (
|
||||||
|
FTS,
|
||||||
|
BTree,
|
||||||
|
Bitmap,
|
||||||
|
HnswSq,
|
||||||
|
IvfFlat,
|
||||||
|
IvfPq,
|
||||||
|
IvfRq,
|
||||||
|
IvfSq,
|
||||||
|
LabelList,
|
||||||
|
)
|
||||||
from lancedb.remote.db import LOOP
|
from lancedb.remote.db import LOOP
|
||||||
import pyarrow as pa
|
import pyarrow as pa
|
||||||
|
|
||||||
@@ -265,6 +275,12 @@ class RemoteTable(Table):
|
|||||||
num_sub_vectors=num_sub_vectors,
|
num_sub_vectors=num_sub_vectors,
|
||||||
num_bits=num_bits,
|
num_bits=num_bits,
|
||||||
)
|
)
|
||||||
|
elif index_type == "IVF_RQ":
|
||||||
|
config = IvfRq(
|
||||||
|
distance_type=metric,
|
||||||
|
num_partitions=num_partitions,
|
||||||
|
num_bits=num_bits,
|
||||||
|
)
|
||||||
elif index_type == "IVF_SQ":
|
elif index_type == "IVF_SQ":
|
||||||
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
|
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
|
||||||
elif index_type == "IVF_HNSW_PQ":
|
elif index_type == "IVF_HNSW_PQ":
|
||||||
@@ -279,7 +295,8 @@ class RemoteTable(Table):
|
|||||||
else:
|
else:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Unknown vector index type: {index_type}. Valid options are"
|
f"Unknown vector index type: {index_type}. Valid options are"
|
||||||
" 'IVF_FLAT', 'IVF_SQ', 'IVF_PQ', 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
|
" 'IVF_FLAT', 'IVF_PQ', 'IVF_RQ', 'IVF_SQ',"
|
||||||
|
" 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
|
||||||
)
|
)
|
||||||
|
|
||||||
LOOP.run(
|
LOOP.run(
|
||||||
|
|||||||
@@ -684,6 +684,24 @@ class Table(ABC):
|
|||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def to_lance(self, **kwargs) -> lance.LanceDataset:
|
||||||
|
"""Return the table as a lance.LanceDataset.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
lance.LanceDataset
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def to_polars(self, **kwargs) -> "pl.DataFrame":
|
||||||
|
"""Return the table as a polars.DataFrame.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
polars.DataFrame
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
def create_index(
|
def create_index(
|
||||||
self,
|
self,
|
||||||
metric="l2",
|
metric="l2",
|
||||||
|
|||||||
@@ -613,6 +613,133 @@ def test_voyageai_multimodal_embedding_text_function():
|
|||||||
assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
|
assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
||||||
|
)
|
||||||
|
def test_voyageai_multimodal_35_embedding_function():
|
||||||
|
"""Test voyage-multimodal-3.5 model with text input."""
|
||||||
|
voyageai = (
|
||||||
|
get_registry()
|
||||||
|
.get("voyageai")
|
||||||
|
.create(name="voyage-multimodal-3.5", max_retries=0)
|
||||||
|
)
|
||||||
|
|
||||||
|
class TextModel(LanceModel):
|
||||||
|
text: str = voyageai.SourceField()
|
||||||
|
vector: Vector(voyageai.ndims()) = voyageai.VectorField()
|
||||||
|
|
||||||
|
df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
|
||||||
|
db = lancedb.connect("~/lancedb")
|
||||||
|
tbl = db.create_table("test_multimodal_35", schema=TextModel, mode="overwrite")
|
||||||
|
|
||||||
|
tbl.add(df)
|
||||||
|
assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
|
||||||
|
assert voyageai.ndims() == 1024
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
||||||
|
)
|
||||||
|
def test_voyageai_multimodal_35_flexible_dimensions():
|
||||||
|
"""Test voyage-multimodal-3.5 model with custom output dimension."""
|
||||||
|
voyageai = (
|
||||||
|
get_registry()
|
||||||
|
.get("voyageai")
|
||||||
|
.create(name="voyage-multimodal-3.5", output_dimension=512, max_retries=0)
|
||||||
|
)
|
||||||
|
|
||||||
|
class TextModel(LanceModel):
|
||||||
|
text: str = voyageai.SourceField()
|
||||||
|
vector: Vector(voyageai.ndims()) = voyageai.VectorField()
|
||||||
|
|
||||||
|
assert voyageai.ndims() == 512
|
||||||
|
|
||||||
|
df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
|
||||||
|
db = lancedb.connect("~/lancedb")
|
||||||
|
tbl = db.create_table("test_multimodal_35_dim", schema=TextModel, mode="overwrite")
|
||||||
|
|
||||||
|
tbl.add(df)
|
||||||
|
assert len(tbl.to_pandas()["vector"][0]) == 512
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
||||||
|
)
|
||||||
|
def test_voyageai_multimodal_35_image_embedding():
|
||||||
|
"""Test voyage-multimodal-3.5 model with image input."""
|
||||||
|
voyageai = (
|
||||||
|
get_registry()
|
||||||
|
.get("voyageai")
|
||||||
|
.create(name="voyage-multimodal-3.5", max_retries=0)
|
||||||
|
)
|
||||||
|
|
||||||
|
class Images(LanceModel):
|
||||||
|
label: str
|
||||||
|
image_uri: str = voyageai.SourceField()
|
||||||
|
vector: Vector(voyageai.ndims()) = voyageai.VectorField()
|
||||||
|
|
||||||
|
db = lancedb.connect("~/lancedb")
|
||||||
|
table = db.create_table(
|
||||||
|
"test_multimodal_35_images", schema=Images, mode="overwrite"
|
||||||
|
)
|
||||||
|
labels = ["cat", "dog"]
|
||||||
|
uris = [
|
||||||
|
"http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
|
||||||
|
"http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
|
||||||
|
]
|
||||||
|
table.add(pd.DataFrame({"label": labels, "image_uri": uris}))
|
||||||
|
assert len(table.to_pandas()["vector"][0]) == voyageai.ndims()
|
||||||
|
assert voyageai.ndims() == 1024
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("dimension", [256, 512, 1024, 2048])
|
||||||
|
def test_voyageai_multimodal_35_all_dimensions(dimension):
|
||||||
|
"""Test voyage-multimodal-3.5 model with all valid output dimensions."""
|
||||||
|
voyageai = (
|
||||||
|
get_registry()
|
||||||
|
.get("voyageai")
|
||||||
|
.create(name="voyage-multimodal-3.5", output_dimension=dimension, max_retries=0)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert voyageai.ndims() == dimension
|
||||||
|
|
||||||
|
class TextModel(LanceModel):
|
||||||
|
text: str = voyageai.SourceField()
|
||||||
|
vector: Vector(voyageai.ndims()) = voyageai.VectorField()
|
||||||
|
|
||||||
|
df = pd.DataFrame({"text": ["hello world"]})
|
||||||
|
db = lancedb.connect("~/lancedb")
|
||||||
|
tbl = db.create_table(
|
||||||
|
f"test_multimodal_35_dim_{dimension}", schema=TextModel, mode="overwrite"
|
||||||
|
)
|
||||||
|
|
||||||
|
tbl.add(df)
|
||||||
|
assert len(tbl.to_pandas()["vector"][0]) == dimension
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.slow
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
|
||||||
|
)
|
||||||
|
def test_voyageai_multimodal_35_invalid_dimension():
|
||||||
|
"""Test voyage-multimodal-3.5 model raises error for invalid output dimension."""
|
||||||
|
with pytest.raises(ValueError, match="Invalid output_dimension"):
|
||||||
|
voyageai = (
|
||||||
|
get_registry()
|
||||||
|
.get("voyageai")
|
||||||
|
.create(name="voyage-multimodal-3.5", output_dimension=999, max_retries=0)
|
||||||
|
)
|
||||||
|
# ndims() is where the validation happens
|
||||||
|
voyageai.ndims()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.slow
|
@pytest.mark.slow
|
||||||
@pytest.mark.skipif(
|
@pytest.mark.skipif(
|
||||||
importlib.util.find_spec("colpali_engine") is None,
|
importlib.util.find_spec("colpali_engine") is None,
|
||||||
|
|||||||
68
python/python/tests/test_s3_bucket_dots.py
Normal file
68
python/python/tests/test_s3_bucket_dots.py
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
|
"""
|
||||||
|
Tests for S3 bucket names containing dots.
|
||||||
|
|
||||||
|
Related issue: https://github.com/lancedb/lancedb/issues/1898
|
||||||
|
|
||||||
|
These tests validate the early error checking for S3 bucket names with dots.
|
||||||
|
No actual S3 connection is made - validation happens before connection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import lancedb
|
||||||
|
|
||||||
|
# Test URIs
|
||||||
|
BUCKET_WITH_DOTS = "s3://my.bucket.name/path"
|
||||||
|
BUCKET_WITH_DOTS_AND_REGION = ("s3://my.bucket.name", {"region": "us-east-1"})
|
||||||
|
BUCKET_WITH_DOTS_AND_AWS_REGION = ("s3://my.bucket.name", {"aws_region": "us-east-1"})
|
||||||
|
BUCKET_WITHOUT_DOTS = "s3://my-bucket/path"
|
||||||
|
|
||||||
|
|
||||||
|
class TestS3BucketWithDotsSync:
|
||||||
|
"""Tests for connect()."""
|
||||||
|
|
||||||
|
def test_bucket_with_dots_requires_region(self):
|
||||||
|
with pytest.raises(ValueError, match="contains dots"):
|
||||||
|
lancedb.connect(BUCKET_WITH_DOTS)
|
||||||
|
|
||||||
|
def test_bucket_with_dots_and_region_passes(self):
|
||||||
|
uri, opts = BUCKET_WITH_DOTS_AND_REGION
|
||||||
|
db = lancedb.connect(uri, storage_options=opts)
|
||||||
|
assert db is not None
|
||||||
|
|
||||||
|
def test_bucket_with_dots_and_aws_region_passes(self):
|
||||||
|
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
|
||||||
|
db = lancedb.connect(uri, storage_options=opts)
|
||||||
|
assert db is not None
|
||||||
|
|
||||||
|
def test_bucket_without_dots_passes(self):
|
||||||
|
db = lancedb.connect(BUCKET_WITHOUT_DOTS)
|
||||||
|
assert db is not None
|
||||||
|
|
||||||
|
|
||||||
|
class TestS3BucketWithDotsAsync:
|
||||||
|
"""Tests for connect_async()."""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bucket_with_dots_requires_region(self):
|
||||||
|
with pytest.raises(ValueError, match="contains dots"):
|
||||||
|
await lancedb.connect_async(BUCKET_WITH_DOTS)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bucket_with_dots_and_region_passes(self):
|
||||||
|
uri, opts = BUCKET_WITH_DOTS_AND_REGION
|
||||||
|
db = await lancedb.connect_async(uri, storage_options=opts)
|
||||||
|
assert db is not None
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bucket_with_dots_and_aws_region_passes(self):
|
||||||
|
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
|
||||||
|
db = await lancedb.connect_async(uri, storage_options=opts)
|
||||||
|
assert db is not None
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bucket_without_dots_passes(self):
|
||||||
|
db = await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
|
||||||
|
assert db is not None
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.23.0"
|
version = "0.23.1"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -1325,25 +1325,27 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_table_names() {
|
async fn test_table_names() {
|
||||||
let tmp_dir = tempdir().unwrap();
|
let tc = new_test_connection().await.unwrap();
|
||||||
|
let db = tc.connection;
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
|
||||||
let mut names = Vec::with_capacity(100);
|
let mut names = Vec::with_capacity(100);
|
||||||
for _ in 0..100 {
|
for _ in 0..100 {
|
||||||
let mut name = uuid::Uuid::new_v4().to_string();
|
let name = uuid::Uuid::new_v4().to_string();
|
||||||
names.push(name.clone());
|
names.push(name.clone());
|
||||||
name.push_str(".lance");
|
db.create_empty_table(name, schema.clone())
|
||||||
create_dir_all(tmp_dir.path().join(&name)).unwrap();
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
names.sort();
|
names.sort();
|
||||||
|
let tables = db.table_names().limit(100).execute().await.unwrap();
|
||||||
let uri = tmp_dir.path().to_str().unwrap();
|
|
||||||
let db = connect(uri).execute().await.unwrap();
|
|
||||||
let tables = db.table_names().execute().await.unwrap();
|
|
||||||
|
|
||||||
assert_eq!(tables, names);
|
assert_eq!(tables, names);
|
||||||
|
|
||||||
let tables = db
|
let tables = db
|
||||||
.table_names()
|
.table_names()
|
||||||
.start_after(&names[30])
|
.start_after(&names[30])
|
||||||
|
.limit(100)
|
||||||
.execute()
|
.execute()
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ use std::collections::HashMap;
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider};
|
|
||||||
use lance_namespace::{
|
use lance_namespace::{
|
||||||
models::{
|
models::{
|
||||||
CreateEmptyTableRequest, CreateNamespaceRequest, CreateNamespaceResponse,
|
CreateEmptyTableRequest, CreateNamespaceRequest, CreateNamespaceResponse,
|
||||||
@@ -19,13 +18,13 @@ use lance_namespace::{
|
|||||||
};
|
};
|
||||||
use lance_namespace_impls::ConnectBuilder;
|
use lance_namespace_impls::ConnectBuilder;
|
||||||
|
|
||||||
use crate::connection::ConnectRequest;
|
|
||||||
use crate::database::ReadConsistency;
|
use crate::database::ReadConsistency;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
|
use crate::table::NativeTable;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
listing::ListingDatabase, BaseTable, CloneTableRequest, CreateTableMode,
|
BaseTable, CloneTableRequest, CreateTableMode, CreateTableRequest as DbCreateTableRequest,
|
||||||
CreateTableRequest as DbCreateTableRequest, Database, OpenTableRequest, TableNamesRequest,
|
Database, OpenTableRequest, TableNamesRequest,
|
||||||
};
|
};
|
||||||
|
|
||||||
/// A database implementation that uses lance-namespace for table management
|
/// A database implementation that uses lance-namespace for table management
|
||||||
@@ -90,51 +89,6 @@ impl std::fmt::Display for LanceNamespaceDatabase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl LanceNamespaceDatabase {
|
|
||||||
/// Create a temporary listing database for the given location
|
|
||||||
///
|
|
||||||
/// Merges storage options with priority: connection < user < namespace
|
|
||||||
async fn create_listing_database(
|
|
||||||
&self,
|
|
||||||
location: &str,
|
|
||||||
table_id: Vec<String>,
|
|
||||||
user_storage_options: Option<&HashMap<String, String>>,
|
|
||||||
response_storage_options: Option<&HashMap<String, String>>,
|
|
||||||
) -> Result<ListingDatabase> {
|
|
||||||
// Merge storage options: connection < user < namespace
|
|
||||||
let mut merged_storage_options = self.storage_options.clone();
|
|
||||||
if let Some(opts) = user_storage_options {
|
|
||||||
merged_storage_options.extend(opts.clone());
|
|
||||||
}
|
|
||||||
if let Some(opts) = response_storage_options {
|
|
||||||
merged_storage_options.extend(opts.clone());
|
|
||||||
}
|
|
||||||
|
|
||||||
let request = ConnectRequest {
|
|
||||||
uri: location.to_string(),
|
|
||||||
#[cfg(feature = "remote")]
|
|
||||||
client_config: Default::default(),
|
|
||||||
options: merged_storage_options,
|
|
||||||
read_consistency_interval: self.read_consistency_interval,
|
|
||||||
session: self.session.clone(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let mut listing_db = ListingDatabase::connect_with_options(&request).await?;
|
|
||||||
|
|
||||||
// Create storage options provider only if namespace returned storage options
|
|
||||||
// (not just user-provided options)
|
|
||||||
if response_storage_options.is_some() {
|
|
||||||
let provider = Arc::new(LanceNamespaceStorageOptionsProvider::new(
|
|
||||||
self.namespace.clone(),
|
|
||||||
table_id,
|
|
||||||
)) as Arc<dyn StorageOptionsProvider>;
|
|
||||||
listing_db.storage_options_provider = Some(provider);
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(listing_db)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl Database for LanceNamespaceDatabase {
|
impl Database for LanceNamespaceDatabase {
|
||||||
fn uri(&self) -> &str {
|
fn uri(&self) -> &str {
|
||||||
@@ -195,14 +149,6 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
// Extract user-provided storage options from request
|
|
||||||
let user_storage_options = request
|
|
||||||
.write_options
|
|
||||||
.lance_write_params
|
|
||||||
.as_ref()
|
|
||||||
.and_then(|lwp| lwp.store_params.as_ref())
|
|
||||||
.and_then(|sp| sp.storage_options.as_ref());
|
|
||||||
|
|
||||||
let mut table_id = request.namespace.clone();
|
let mut table_id = request.namespace.clone();
|
||||||
table_id.push(request.name.clone());
|
table_id.push(request.name.clone());
|
||||||
let describe_request = DescribeTableRequest {
|
let describe_request = DescribeTableRequest {
|
||||||
@@ -235,34 +181,20 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
CreateTableMode::ExistOk(_) => {
|
CreateTableMode::ExistOk(_) => {
|
||||||
if let Ok(response) = describe_result {
|
if describe_result.is_ok() {
|
||||||
let location = response.location.ok_or_else(|| Error::Runtime {
|
let native_table = NativeTable::open_from_namespace(
|
||||||
message: "Table location is missing from namespace response".to_string(),
|
self.namespace.clone(),
|
||||||
})?;
|
&request.name,
|
||||||
|
request.namespace.clone(),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
self.read_consistency_interval,
|
||||||
|
self.server_side_query_enabled,
|
||||||
|
self.session.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let listing_db = self
|
return Ok(Arc::new(native_table));
|
||||||
.create_listing_database(
|
|
||||||
&location,
|
|
||||||
table_id.clone(),
|
|
||||||
user_storage_options,
|
|
||||||
response.storage_options.as_ref(),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let namespace_client = self
|
|
||||||
.server_side_query_enabled
|
|
||||||
.then(|| self.namespace.clone());
|
|
||||||
|
|
||||||
return listing_db
|
|
||||||
.open_table(OpenTableRequest {
|
|
||||||
name: request.name.clone(),
|
|
||||||
namespace: request.namespace.clone(),
|
|
||||||
index_cache_size: None,
|
|
||||||
lance_read_params: None,
|
|
||||||
location: Some(location),
|
|
||||||
namespace_client,
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -294,82 +226,37 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
message: "Table location is missing from create_empty_table response".to_string(),
|
message: "Table location is missing from create_empty_table response".to_string(),
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let listing_db = self
|
let native_table = NativeTable::create_from_namespace(
|
||||||
.create_listing_database(
|
self.namespace.clone(),
|
||||||
&location,
|
&location,
|
||||||
table_id.clone(),
|
&request.name,
|
||||||
user_storage_options,
|
request.namespace.clone(),
|
||||||
create_empty_response.storage_options.as_ref(),
|
request.data,
|
||||||
)
|
None, // write_store_wrapper not used for namespace connections
|
||||||
.await?;
|
request.write_options.lance_write_params,
|
||||||
|
self.read_consistency_interval,
|
||||||
|
self.server_side_query_enabled,
|
||||||
|
self.session.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let namespace_client = self
|
Ok(Arc::new(native_table))
|
||||||
.server_side_query_enabled
|
|
||||||
.then(|| self.namespace.clone());
|
|
||||||
|
|
||||||
let create_request = DbCreateTableRequest {
|
|
||||||
name: request.name,
|
|
||||||
namespace: request.namespace,
|
|
||||||
data: request.data,
|
|
||||||
mode: request.mode,
|
|
||||||
write_options: request.write_options,
|
|
||||||
location: Some(location),
|
|
||||||
namespace_client,
|
|
||||||
};
|
|
||||||
|
|
||||||
listing_db.create_table(create_request).await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
// Extract user-provided storage options from request
|
let native_table = NativeTable::open_from_namespace(
|
||||||
let user_storage_options = request
|
self.namespace.clone(),
|
||||||
.lance_read_params
|
&request.name,
|
||||||
.as_ref()
|
request.namespace.clone(),
|
||||||
.and_then(|lrp| lrp.store_options.as_ref())
|
None, // write_store_wrapper not used for namespace connections
|
||||||
.and_then(|so| so.storage_options.as_ref());
|
request.lance_read_params,
|
||||||
|
self.read_consistency_interval,
|
||||||
|
self.server_side_query_enabled,
|
||||||
|
self.session.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let mut table_id = request.namespace.clone();
|
Ok(Arc::new(native_table))
|
||||||
table_id.push(request.name.clone());
|
|
||||||
|
|
||||||
let describe_request = DescribeTableRequest {
|
|
||||||
id: Some(table_id.clone()),
|
|
||||||
version: None,
|
|
||||||
};
|
|
||||||
let response = self
|
|
||||||
.namespace
|
|
||||||
.describe_table(describe_request)
|
|
||||||
.await
|
|
||||||
.map_err(|e| Error::Runtime {
|
|
||||||
message: format!("Failed to describe table: {}", e),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let location = response.location.ok_or_else(|| Error::Runtime {
|
|
||||||
message: "Table location is missing from namespace response".to_string(),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let listing_db = self
|
|
||||||
.create_listing_database(
|
|
||||||
&location,
|
|
||||||
table_id.clone(),
|
|
||||||
user_storage_options,
|
|
||||||
response.storage_options.as_ref(),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let namespace_client = self
|
|
||||||
.server_side_query_enabled
|
|
||||||
.then(|| self.namespace.clone());
|
|
||||||
|
|
||||||
let open_request = OpenTableRequest {
|
|
||||||
name: request.name.clone(),
|
|
||||||
namespace: request.namespace.clone(),
|
|
||||||
index_cache_size: request.index_cache_size,
|
|
||||||
lance_read_params: request.lance_read_params,
|
|
||||||
location: Some(location),
|
|
||||||
namespace_client,
|
|
||||||
};
|
|
||||||
|
|
||||||
listing_db.open_table(open_request).await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn clone_table(&self, _request: CloneTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn clone_table(&self, _request: CloneTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
|
|||||||
@@ -1088,6 +1088,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
|||||||
body["num_partitions"] = serde_json::Value::Number(num_partitions.into());
|
body["num_partitions"] = serde_json::Value::Number(num_partitions.into());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Index::IvfRq(index) => {
|
||||||
|
body[INDEX_TYPE_KEY] = serde_json::Value::String("IVF_RQ".to_string());
|
||||||
|
body[METRIC_TYPE_KEY] =
|
||||||
|
serde_json::Value::String(index.distance_type.to_string().to_lowercase());
|
||||||
|
if let Some(num_partitions) = index.num_partitions {
|
||||||
|
body["num_partitions"] = serde_json::Value::Number(num_partitions.into());
|
||||||
|
}
|
||||||
|
if let Some(num_bits) = index.num_bits {
|
||||||
|
body["num_bits"] = serde_json::Value::Number(num_bits.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
Index::BTree(_) => {
|
Index::BTree(_) => {
|
||||||
body[INDEX_TYPE_KEY] = serde_json::Value::String("BTREE".to_string());
|
body[INDEX_TYPE_KEY] = serde_json::Value::String("BTREE".to_string());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ use lance::dataset::{
|
|||||||
use lance::dataset::{MergeInsertBuilder as LanceMergeInsertBuilder, WhenNotMatchedBySource};
|
use lance::dataset::{MergeInsertBuilder as LanceMergeInsertBuilder, WhenNotMatchedBySource};
|
||||||
use lance::index::vector::utils::infer_vector_dim;
|
use lance::index::vector::utils::infer_vector_dim;
|
||||||
use lance::index::vector::VectorIndexParams;
|
use lance::index::vector::VectorIndexParams;
|
||||||
use lance::io::WrappingObjectStore;
|
use lance::io::{ObjectStoreParams, WrappingObjectStore};
|
||||||
use lance_datafusion::exec::{analyze_plan as lance_analyze_plan, execute_plan};
|
use lance_datafusion::exec::{analyze_plan as lance_analyze_plan, execute_plan};
|
||||||
use lance_datafusion::utils::StreamingWriteSource;
|
use lance_datafusion::utils::StreamingWriteSource;
|
||||||
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
|
use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
|
||||||
@@ -40,6 +40,7 @@ use lance_index::vector::pq::PQBuildParams;
|
|||||||
use lance_index::vector::sq::builder::SQBuildParams;
|
use lance_index::vector::sq::builder::SQBuildParams;
|
||||||
use lance_index::DatasetIndexExt;
|
use lance_index::DatasetIndexExt;
|
||||||
use lance_index::IndexType;
|
use lance_index::IndexType;
|
||||||
|
use lance_io::object_store::LanceNamespaceStorageOptionsProvider;
|
||||||
use lance_namespace::models::{
|
use lance_namespace::models::{
|
||||||
QueryTableRequest as NsQueryTableRequest, QueryTableRequestFullTextQuery,
|
QueryTableRequest as NsQueryTableRequest, QueryTableRequestFullTextQuery,
|
||||||
QueryTableRequestVector, StringFtsQuery,
|
QueryTableRequestVector, StringFtsQuery,
|
||||||
@@ -1611,6 +1612,105 @@ impl NativeTable {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Opens an existing Table using a namespace client.
|
||||||
|
///
|
||||||
|
/// This method uses `DatasetBuilder::from_namespace` to open the table, which
|
||||||
|
/// automatically fetches the table location and storage options from the namespace.
|
||||||
|
/// This eliminates the need to pre-fetch and merge storage options before opening.
|
||||||
|
///
|
||||||
|
/// # Arguments
|
||||||
|
///
|
||||||
|
/// * `namespace_client` - The namespace client to use for fetching table metadata
|
||||||
|
/// * `name` - The table name
|
||||||
|
/// * `namespace` - The namespace path (e.g., vec!["parent", "child"])
|
||||||
|
/// * `write_store_wrapper` - Optional wrapper for the object store on write path
|
||||||
|
/// * `params` - Optional read parameters
|
||||||
|
/// * `read_consistency_interval` - Optional interval for read consistency
|
||||||
|
/// * `server_side_query_enabled` - Whether to enable server-side query execution.
|
||||||
|
/// When true, the namespace_client will be stored and queries will be executed
|
||||||
|
/// on the namespace server. When false, the namespace is only used for opening
|
||||||
|
/// the table, and queries are executed locally.
|
||||||
|
/// * `session` - Optional session for object stores and caching
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
///
|
||||||
|
/// * A [NativeTable] object.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub async fn open_from_namespace(
|
||||||
|
namespace_client: Arc<dyn LanceNamespace>,
|
||||||
|
name: &str,
|
||||||
|
namespace: Vec<String>,
|
||||||
|
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||||
|
params: Option<ReadParams>,
|
||||||
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
|
server_side_query_enabled: bool,
|
||||||
|
session: Option<Arc<lance::session::Session>>,
|
||||||
|
) -> Result<Self> {
|
||||||
|
let mut params = params.unwrap_or_default();
|
||||||
|
|
||||||
|
// Set the session in read params
|
||||||
|
if let Some(sess) = session {
|
||||||
|
params.session(sess);
|
||||||
|
}
|
||||||
|
|
||||||
|
// patch the params if we have a write store wrapper
|
||||||
|
let params = match write_store_wrapper.clone() {
|
||||||
|
Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
|
||||||
|
None => params,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Build table_id from namespace + name
|
||||||
|
let mut table_id = namespace.clone();
|
||||||
|
table_id.push(name.to_string());
|
||||||
|
|
||||||
|
// Use DatasetBuilder::from_namespace which automatically fetches location
|
||||||
|
// and storage options from the namespace
|
||||||
|
let builder = DatasetBuilder::from_namespace(
|
||||||
|
namespace_client.clone(),
|
||||||
|
table_id,
|
||||||
|
false, // Don't ignore namespace storage options
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| match e {
|
||||||
|
lance::Error::Namespace { source, .. } => Error::Runtime {
|
||||||
|
message: format!("Failed to get table info from namespace: {:?}", source),
|
||||||
|
},
|
||||||
|
source => Error::Lance { source },
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let dataset = builder
|
||||||
|
.with_read_params(params)
|
||||||
|
.load()
|
||||||
|
.await
|
||||||
|
.map_err(|e| match e {
|
||||||
|
lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
|
||||||
|
name: name.to_string(),
|
||||||
|
source: Box::new(e),
|
||||||
|
},
|
||||||
|
source => Error::Lance { source },
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let uri = dataset.uri().to_string();
|
||||||
|
let dataset = DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval);
|
||||||
|
let id = Self::build_id(&namespace, name);
|
||||||
|
|
||||||
|
let stored_namespace_client = if server_side_query_enabled {
|
||||||
|
Some(namespace_client)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
namespace,
|
||||||
|
id,
|
||||||
|
uri,
|
||||||
|
dataset,
|
||||||
|
read_consistency_interval,
|
||||||
|
namespace_client: stored_namespace_client,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn get_table_name(uri: &str) -> Result<String> {
|
fn get_table_name(uri: &str) -> Result<String> {
|
||||||
let path = Path::new(uri);
|
let path = Path::new(uri);
|
||||||
let name = path
|
let name = path
|
||||||
@@ -1722,6 +1822,102 @@ impl NativeTable {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Creates a new Table using a namespace client for storage options.
|
||||||
|
///
|
||||||
|
/// This method sets up a `StorageOptionsProvider` from the namespace client,
|
||||||
|
/// enabling automatic credential refresh for cloud storage. The namespace
|
||||||
|
/// is used for:
|
||||||
|
/// 1. Setting up storage options provider for credential vending
|
||||||
|
/// 2. Optionally enabling server-side query execution
|
||||||
|
///
|
||||||
|
/// # Arguments
|
||||||
|
///
|
||||||
|
/// * `namespace_client` - The namespace client to use for storage options
|
||||||
|
/// * `uri` - The URI to the table (obtained from create_empty_table response)
|
||||||
|
/// * `name` - The table name
|
||||||
|
/// * `namespace` - The namespace path (e.g., vec!["parent", "child"])
|
||||||
|
/// * `batches` - RecordBatch to be saved in the database
|
||||||
|
/// * `write_store_wrapper` - Optional wrapper for the object store on write path
|
||||||
|
/// * `params` - Optional write parameters
|
||||||
|
/// * `read_consistency_interval` - Optional interval for read consistency
|
||||||
|
/// * `server_side_query_enabled` - Whether to enable server-side query execution
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
///
|
||||||
|
/// * A [NativeTable] object.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub async fn create_from_namespace(
|
||||||
|
namespace_client: Arc<dyn LanceNamespace>,
|
||||||
|
uri: &str,
|
||||||
|
name: &str,
|
||||||
|
namespace: Vec<String>,
|
||||||
|
batches: impl StreamingWriteSource,
|
||||||
|
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||||
|
params: Option<WriteParams>,
|
||||||
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
|
server_side_query_enabled: bool,
|
||||||
|
session: Option<Arc<lance::session::Session>>,
|
||||||
|
) -> Result<Self> {
|
||||||
|
// Build table_id from namespace + name for the storage options provider
|
||||||
|
let mut table_id = namespace.clone();
|
||||||
|
table_id.push(name.to_string());
|
||||||
|
|
||||||
|
// Set up storage options provider from namespace
|
||||||
|
let storage_options_provider = Arc::new(LanceNamespaceStorageOptionsProvider::new(
|
||||||
|
namespace_client.clone(),
|
||||||
|
table_id,
|
||||||
|
));
|
||||||
|
|
||||||
|
// Start with provided params or defaults
|
||||||
|
let mut params = params.unwrap_or_default();
|
||||||
|
|
||||||
|
// Set the session in write params
|
||||||
|
if let Some(sess) = session {
|
||||||
|
params.session = Some(sess);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure store_params exists and set the storage options provider
|
||||||
|
let store_params = params
|
||||||
|
.store_params
|
||||||
|
.get_or_insert_with(ObjectStoreParams::default);
|
||||||
|
store_params.storage_options_provider = Some(storage_options_provider);
|
||||||
|
|
||||||
|
// Patch the params if we have a write store wrapper
|
||||||
|
let params = match write_store_wrapper.clone() {
|
||||||
|
Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
|
||||||
|
None => params,
|
||||||
|
};
|
||||||
|
|
||||||
|
let insert_builder = InsertBuilder::new(uri).with_params(¶ms);
|
||||||
|
let dataset = insert_builder
|
||||||
|
.execute_stream(batches)
|
||||||
|
.await
|
||||||
|
.map_err(|e| match e {
|
||||||
|
lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
|
||||||
|
name: name.to_string(),
|
||||||
|
},
|
||||||
|
source => Error::Lance { source },
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let id = Self::build_id(&namespace, name);
|
||||||
|
|
||||||
|
let stored_namespace_client = if server_side_query_enabled {
|
||||||
|
Some(namespace_client)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
name: name.to_string(),
|
||||||
|
namespace,
|
||||||
|
id,
|
||||||
|
uri: uri.to_string(),
|
||||||
|
dataset: DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval),
|
||||||
|
read_consistency_interval,
|
||||||
|
namespace_client: stored_namespace_client,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn optimize_indices(&self, options: &OptimizeOptions) -> Result<()> {
|
async fn optimize_indices(&self, options: &OptimizeOptions) -> Result<()> {
|
||||||
info!("LanceDB: optimizing indices: {:?}", options);
|
info!("LanceDB: optimizing indices: {:?}", options);
|
||||||
self.dataset
|
self.dataset
|
||||||
|
|||||||
@@ -5,16 +5,19 @@
|
|||||||
|
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::io::{BufRead, BufReader};
|
use std::process::Stdio;
|
||||||
use std::process::{Child, ChildStdout, Command, Stdio};
|
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||||
|
use tokio::process::{Child, ChildStdout, Command};
|
||||||
|
use tokio::sync::mpsc;
|
||||||
|
|
||||||
use crate::{connect, Connection};
|
use crate::{connect, Connection};
|
||||||
use anyhow::{bail, Result};
|
use anyhow::{anyhow, bail, Result};
|
||||||
use tempfile::{tempdir, TempDir};
|
use tempfile::{tempdir, TempDir};
|
||||||
|
|
||||||
pub struct TestConnection {
|
pub struct TestConnection {
|
||||||
pub uri: String,
|
pub uri: String,
|
||||||
pub connection: Connection,
|
pub connection: Connection,
|
||||||
|
pub is_remote: bool,
|
||||||
_temp_dir: Option<TempDir>,
|
_temp_dir: Option<TempDir>,
|
||||||
_process: Option<TestProcess>,
|
_process: Option<TestProcess>,
|
||||||
}
|
}
|
||||||
@@ -37,6 +40,56 @@ pub async fn new_test_connection() -> Result<TestConnection> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn spawn_stdout_reader(
|
||||||
|
mut stdout: BufReader<ChildStdout>,
|
||||||
|
port_sender: mpsc::Sender<anyhow::Result<String>>,
|
||||||
|
) -> tokio::task::JoinHandle<()> {
|
||||||
|
let print_stdout = env::var("PRINT_LANCEDB_TEST_CONNECTION_SCRIPT_OUTPUT").is_ok();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut line = String::new();
|
||||||
|
let re = Regex::new(r"Query node now listening on 0.0.0.0:(.*)").unwrap();
|
||||||
|
loop {
|
||||||
|
line.clear();
|
||||||
|
let result = stdout.read_line(&mut line).await;
|
||||||
|
if let Err(err) = result {
|
||||||
|
port_sender
|
||||||
|
.send(Err(anyhow!(
|
||||||
|
"error while reading from process output: {}",
|
||||||
|
err
|
||||||
|
)))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
return;
|
||||||
|
} else if result.unwrap() == 0 {
|
||||||
|
port_sender
|
||||||
|
.send(Err(anyhow!(
|
||||||
|
" hit EOF before reading port from process output."
|
||||||
|
)))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if re.is_match(&line) {
|
||||||
|
let caps = re.captures(&line).unwrap();
|
||||||
|
port_sender.send(Ok(caps[1].to_string())).await.unwrap();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
loop {
|
||||||
|
line.clear();
|
||||||
|
match stdout.read_line(&mut line).await {
|
||||||
|
Err(_) => return,
|
||||||
|
Ok(0) => return,
|
||||||
|
Ok(_size) => {
|
||||||
|
if print_stdout {
|
||||||
|
print!("{}", line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn new_remote_connection(script_path: &str) -> Result<TestConnection> {
|
async fn new_remote_connection(script_path: &str) -> Result<TestConnection> {
|
||||||
let temp_dir = tempdir()?;
|
let temp_dir = tempdir()?;
|
||||||
let data_path = temp_dir.path().to_str().unwrap().to_string();
|
let data_path = temp_dir.path().to_str().unwrap().to_string();
|
||||||
@@ -57,38 +110,25 @@ async fn new_remote_connection(script_path: &str) -> Result<TestConnection> {
|
|||||||
child: child_result.unwrap(),
|
child: child_result.unwrap(),
|
||||||
};
|
};
|
||||||
let stdout = BufReader::new(process.child.stdout.take().unwrap());
|
let stdout = BufReader::new(process.child.stdout.take().unwrap());
|
||||||
let port = read_process_port(stdout)?;
|
let (port_sender, mut port_receiver) = mpsc::channel(5);
|
||||||
|
let _reader = spawn_stdout_reader(stdout, port_sender).await;
|
||||||
|
let port = match port_receiver.recv().await {
|
||||||
|
None => bail!("Unable to determine the port number used by the phalanx process we spawned, because the reader thread was closed too soon."),
|
||||||
|
Some(Err(err)) => bail!("Unable to determine the port number used by the phalanx process we spawned, because of an error, {}", err),
|
||||||
|
Some(Ok(port)) => port,
|
||||||
|
};
|
||||||
let uri = "db://test";
|
let uri = "db://test";
|
||||||
let host_override = format!("http://localhost:{}", port);
|
let host_override = format!("http://localhost:{}", port);
|
||||||
let connection = create_new_connection(uri, &host_override).await?;
|
let connection = create_new_connection(uri, &host_override).await?;
|
||||||
Ok(TestConnection {
|
Ok(TestConnection {
|
||||||
uri: uri.to_string(),
|
uri: uri.to_string(),
|
||||||
connection,
|
connection,
|
||||||
|
is_remote: true,
|
||||||
_temp_dir: Some(temp_dir),
|
_temp_dir: Some(temp_dir),
|
||||||
_process: Some(process),
|
_process: Some(process),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn read_process_port(mut stdout: BufReader<ChildStdout>) -> Result<String> {
|
|
||||||
let mut line = String::new();
|
|
||||||
let re = Regex::new(r"Query node now listening on 0.0.0.0:(.*)").unwrap();
|
|
||||||
loop {
|
|
||||||
let result = stdout.read_line(&mut line);
|
|
||||||
if let Err(err) = result {
|
|
||||||
bail!(format!(
|
|
||||||
"read_process_port: error while reading from process output: {}",
|
|
||||||
err
|
|
||||||
));
|
|
||||||
} else if result.unwrap() == 0 {
|
|
||||||
bail!("read_process_port: hit EOF before reading port from process output.");
|
|
||||||
}
|
|
||||||
if re.is_match(&line) {
|
|
||||||
let caps = re.captures(&line).unwrap();
|
|
||||||
return Ok(caps[1].to_string());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(feature = "remote")]
|
#[cfg(feature = "remote")]
|
||||||
async fn create_new_connection(uri: &str, host_override: &str) -> crate::error::Result<Connection> {
|
async fn create_new_connection(uri: &str, host_override: &str) -> crate::error::Result<Connection> {
|
||||||
connect(uri)
|
connect(uri)
|
||||||
@@ -114,6 +154,7 @@ async fn new_local_connection() -> Result<TestConnection> {
|
|||||||
Ok(TestConnection {
|
Ok(TestConnection {
|
||||||
uri: uri.to_string(),
|
uri: uri.to_string(),
|
||||||
connection,
|
connection,
|
||||||
|
is_remote: false,
|
||||||
_temp_dir: Some(temp_dir),
|
_temp_dir: Some(temp_dir),
|
||||||
_process: None,
|
_process: None,
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user