mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-23 05:19:58 +00:00
Compare commits
11 Commits
v0.23.0-be
...
python-v0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ba94e69d5d | ||
|
|
9e60fda0ec | ||
|
|
3e0d451e9b | ||
|
|
94bdffe13c | ||
|
|
b93ea3a388 | ||
|
|
ff20d12f20 | ||
|
|
5f3e133470 | ||
|
|
332e722a64 | ||
|
|
3f63c4f8d9 | ||
|
|
39a18baf59 | ||
|
|
0960e19559 |
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.23.0-beta.0"
|
||||
current_version = "0.23.0"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
4
.github/workflows/nodejs.yml
vendored
4
.github/workflows/nodejs.yml
vendored
@@ -88,7 +88,7 @@ jobs:
|
||||
npm install -g @napi-rs/cli
|
||||
- name: Build
|
||||
run: |
|
||||
npm ci
|
||||
npm ci --include=optional
|
||||
npm run build:debug -- --profile ci
|
||||
npm run tsc
|
||||
- name: Setup localstack
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
npm install -g @napi-rs/cli
|
||||
- name: Build
|
||||
run: |
|
||||
npm ci
|
||||
npm ci --include=optional
|
||||
npm run build:debug -- --profile ci
|
||||
npm run tsc
|
||||
- name: Test
|
||||
|
||||
6
.github/workflows/python.yml
vendored
6
.github/workflows/python.yml
vendored
@@ -49,8 +49,8 @@ jobs:
|
||||
|
||||
type-check:
|
||||
name: "Type Check"
|
||||
timeout-minutes: 30
|
||||
runs-on: "ubuntu-22.04"
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-2404-8x-x64
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
doctest:
|
||||
name: "Doctest"
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-2404-8x-x64
|
||||
defaults:
|
||||
run:
|
||||
|
||||
172
Cargo.lock
generated
172
Cargo.lock
generated
@@ -1041,6 +1041,61 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.7.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.7.0",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"sync_wrapper",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backoff"
|
||||
version = "0.4.0"
|
||||
@@ -3086,8 +3141,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.2",
|
||||
@@ -3930,6 +3985,7 @@ dependencies = [
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
@@ -4422,8 +4478,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4488,8 +4544,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4507,8 +4563,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"paste",
|
||||
@@ -4517,8 +4573,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4554,8 +4610,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4585,8 +4641,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4603,8 +4659,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4641,8 +4697,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4674,8 +4730,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-types",
|
||||
@@ -4686,8 +4742,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4748,8 +4804,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4789,8 +4845,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4806,8 +4862,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -4819,13 +4875,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
"arrow-schema",
|
||||
"async-trait",
|
||||
"axum",
|
||||
"bytes",
|
||||
"futures",
|
||||
"lance",
|
||||
@@ -4837,9 +4894,12 @@ dependencies = [
|
||||
"object_store",
|
||||
"rand 0.9.2",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-http 0.5.2",
|
||||
"url",
|
||||
]
|
||||
|
||||
@@ -4858,8 +4918,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4898,8 +4958,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "1.1.0-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.1.0-beta.1#ddea38f049e64df8b893e1c8ecca7878ea373d1e"
|
||||
version = "1.0.1-beta.1"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4910,7 +4970,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.23.0-beta.0"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -4989,7 +5049,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.23.0-beta.0"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-ipc",
|
||||
@@ -5009,7 +5069,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.26.0-beta.0"
|
||||
version = "0.26.0"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5277,6 +5337,12 @@ dependencies = [
|
||||
"regex-automata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
|
||||
|
||||
[[package]]
|
||||
name = "matrixmultiply"
|
||||
version = "0.3.10"
|
||||
@@ -7265,7 +7331,7 @@ dependencies = [
|
||||
"tokio-rustls 0.26.4",
|
||||
"tokio-util",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tower-http 0.6.6",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
@@ -7784,6 +7850,17 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_path_to_error"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_plain"
|
||||
version = "1.0.2"
|
||||
@@ -8819,6 +8896,24 @@ dependencies = [
|
||||
"tokio",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower-http"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
|
||||
dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
"bytes",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8857,6 +8952,7 @@ version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
||||
dependencies = [
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
|
||||
28
Cargo.toml
28
Cargo.toml
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.78.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=1.1.0-beta.1", default-features = false, "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=1.1.0-beta.1", default-features = false, "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=1.1.0-beta.1", default-features = false, "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=1.1.0-beta.1", "tag" = "v1.1.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "56.2", optional = false }
|
||||
|
||||
@@ -229,6 +229,29 @@ def set_local_version():
|
||||
update_cargo_toml(line_updater)
|
||||
|
||||
|
||||
def update_lockfiles(version: str, fallback_to_git: bool = False):
|
||||
"""
|
||||
Update Cargo metadata and optionally fall back to using the git tag if the
|
||||
requested crates.io version is unavailable.
|
||||
"""
|
||||
try:
|
||||
print("Updating lockfiles...", file=sys.stderr, end="")
|
||||
run_command("cargo metadata > /dev/null")
|
||||
print(" done.", file=sys.stderr)
|
||||
except Exception as e:
|
||||
if fallback_to_git and "failed to select a version" in str(e):
|
||||
print(
|
||||
f" failed for crates.io v{version}, retrying with git tag...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
set_preview_version(version)
|
||||
print("Updating lockfiles...", file=sys.stderr, end="")
|
||||
run_command("cargo metadata > /dev/null")
|
||||
print(" done.", file=sys.stderr)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description="Set the version of the Lance package.")
|
||||
parser.add_argument(
|
||||
"version",
|
||||
@@ -244,6 +267,7 @@ if args.version == "stable":
|
||||
file=sys.stderr,
|
||||
)
|
||||
set_stable_version(latest_stable_version)
|
||||
update_lockfiles(latest_stable_version)
|
||||
elif args.version == "preview":
|
||||
latest_preview_version = get_latest_preview_version()
|
||||
print(
|
||||
@@ -251,8 +275,10 @@ elif args.version == "preview":
|
||||
file=sys.stderr,
|
||||
)
|
||||
set_preview_version(latest_preview_version)
|
||||
update_lockfiles(latest_preview_version)
|
||||
elif args.version == "local":
|
||||
set_local_version()
|
||||
update_lockfiles("local")
|
||||
else:
|
||||
# Parse the version number.
|
||||
version = args.version
|
||||
@@ -262,9 +288,7 @@ else:
|
||||
|
||||
if "beta" in version:
|
||||
set_preview_version(version)
|
||||
update_lockfiles(version)
|
||||
else:
|
||||
set_stable_version(version)
|
||||
|
||||
print("Updating lockfiles...", file=sys.stderr, end="")
|
||||
run_command("cargo metadata > /dev/null")
|
||||
print(" done.", file=sys.stderr)
|
||||
update_lockfiles(version, fallback_to_git=True)
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.23.0-beta.0</version>
|
||||
<version>0.23.0</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.23.0-beta.0</version>
|
||||
<version>0.23.0-final.0</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.23.0-beta.0</version>
|
||||
<version>0.23.0-final.0</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.23.0-beta.0"
|
||||
version = "0.23.0"
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-x64",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.darwin-x64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
4
nodejs/package-lock.json
generated
4
nodejs/package-lock.json
generated
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.23.0-beta.0",
|
||||
"version": "0.23.0",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.26.0-beta.1"
|
||||
current_version = "0.26.1-beta.0"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.26.0-beta.1"
|
||||
version = "0.26.1-beta.0"
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
license.workspace = true
|
||||
|
||||
@@ -10,7 +10,7 @@ dependencies = [
|
||||
"pyarrow>=16",
|
||||
"pydantic>=1.10",
|
||||
"tqdm>=4.27.0",
|
||||
"lance-namespace>=0.2.1"
|
||||
"lance-namespace>=0.3.2"
|
||||
]
|
||||
description = "lancedb"
|
||||
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
|
||||
|
||||
@@ -3208,7 +3208,27 @@ def _infer_target_schema(
|
||||
if pa.types.is_floating(field.type.value_type):
|
||||
target_type = pa.list_(pa.float32(), dim)
|
||||
elif pa.types.is_integer(field.type.value_type):
|
||||
target_type = pa.list_(pa.uint8(), dim)
|
||||
values = peeked.column(i)
|
||||
|
||||
if isinstance(values, pa.ChunkedArray):
|
||||
values = values.combine_chunks()
|
||||
|
||||
flattened = values.flatten()
|
||||
valid_count = pc.count(flattened, mode="only_valid").as_py()
|
||||
|
||||
if valid_count == 0:
|
||||
target_type = pa.list_(pa.uint8(), dim)
|
||||
else:
|
||||
min_max = pc.min_max(flattened)
|
||||
min_value = min_max["min"].as_py()
|
||||
max_value = min_max["max"].as_py()
|
||||
|
||||
if (min_value is not None and min_value < 0) or (
|
||||
max_value is not None and max_value > 255
|
||||
):
|
||||
target_type = pa.list_(pa.float32(), dim)
|
||||
else:
|
||||
target_type = pa.list_(pa.uint8(), dim)
|
||||
else:
|
||||
continue # Skip non-numeric types
|
||||
|
||||
|
||||
@@ -46,6 +46,39 @@ def test_basic(mem_db: DBConnection):
|
||||
assert table.to_arrow() == expected_data
|
||||
|
||||
|
||||
def test_create_table_infers_large_int_vectors(mem_db: DBConnection):
|
||||
data = [{"vector": [0, 300]}]
|
||||
|
||||
table = mem_db.create_table(
|
||||
"int_vector_overflow", data=data, mode="overwrite", exist_ok=True
|
||||
)
|
||||
|
||||
vector_field = table.schema.field("vector")
|
||||
assert vector_field.type == pa.list_(pa.float32(), 2)
|
||||
|
||||
vector_column = table.to_arrow().column("vector")
|
||||
assert vector_column.type == pa.list_(pa.float32(), 2)
|
||||
assert vector_column.to_pylist() == [[0.0, 300.0]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_table_async_infers_large_int_vectors(
|
||||
mem_db_async: AsyncConnection,
|
||||
):
|
||||
data = [{"vector": [256, 257]}]
|
||||
|
||||
table = await mem_db_async.create_table(
|
||||
"int_vector_overflow_async", data=data, mode="overwrite", exist_ok=True
|
||||
)
|
||||
|
||||
schema = await table.schema()
|
||||
assert schema.field("vector").type == pa.list_(pa.float32(), 2)
|
||||
|
||||
vector_column = (await table.to_arrow()).column("vector")
|
||||
assert vector_column.type == pa.list_(pa.float32(), 2)
|
||||
assert vector_column.to_pylist() == [[256.0, 257.0]]
|
||||
|
||||
|
||||
def test_input_data_type(mem_db: DBConnection, tmp_path):
|
||||
schema = pa.schema(
|
||||
{
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.23.0-beta.0"
|
||||
version = "0.23.0"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
@@ -110,7 +110,7 @@ oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
|
||||
gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
|
||||
azure = ["lance/azure", "lance-io/azure", "lance-namespace-impls/dir-azure"]
|
||||
dynamodb = ["lance/dynamodb", "aws"]
|
||||
remote = ["dep:reqwest", "dep:http", "lance-namespace-impls/rest"]
|
||||
remote = ["dep:reqwest", "dep:http", "lance-namespace-impls/rest", "lance-namespace-impls/rest-adapter"]
|
||||
fp16kernels = ["lance-linalg/fp16kernels"]
|
||||
s3-test = []
|
||||
bedrock = ["dep:aws-sdk-bedrockruntime"]
|
||||
|
||||
@@ -804,6 +804,14 @@ impl Connection {
|
||||
self.internal.describe_namespace(request).await
|
||||
}
|
||||
|
||||
/// Get the equivalent namespace client in the database of this connection.
|
||||
/// For LanceNamespaceDatabase, it is the underlying LanceNamespace.
|
||||
/// For ListingDatabase, it is the equivalent DirectoryNamespace.
|
||||
/// For RemoteDatabase, it is the equivalent RestNamespace.
|
||||
pub async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> {
|
||||
self.internal.namespace_client().await
|
||||
}
|
||||
|
||||
/// List tables with pagination support
|
||||
pub async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
|
||||
self.internal.list_tables(request).await
|
||||
|
||||
@@ -296,4 +296,10 @@ pub trait Database:
|
||||
/// Drop all tables in the database
|
||||
async fn drop_all_tables(&self, namespace: &[String]) -> Result<()>;
|
||||
fn as_any(&self) -> &dyn std::any::Any;
|
||||
|
||||
/// Get the equivalent namespace client of this database
|
||||
/// For LanceNamespaceDatabase, it is the underlying LanceNamespace.
|
||||
/// For ListingDatabase, it is the equivalent DirectoryNamespace.
|
||||
/// For RemoteDatabase, it is the equivalent RestNamespace.
|
||||
async fn namespace_client(&self) -> Result<Arc<dyn LanceNamespace>>;
|
||||
}
|
||||
|
||||
@@ -1043,6 +1043,24 @@ impl Database for ListingDatabase {
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> {
|
||||
// Create a DirectoryNamespace pointing to the same root with the same storage options
|
||||
let mut builder = lance_namespace_impls::DirectoryNamespaceBuilder::new(&self.uri);
|
||||
|
||||
// Add storage options
|
||||
if !self.storage_options.is_empty() {
|
||||
builder = builder.storage_options(self.storage_options.clone());
|
||||
}
|
||||
|
||||
// Use the same session
|
||||
builder = builder.session(self.session.clone());
|
||||
|
||||
let namespace = builder.build().await.map_err(|e| Error::Runtime {
|
||||
message: format!("Failed to create namespace client: {}", e),
|
||||
})?;
|
||||
Ok(Arc::new(namespace) as Arc<dyn lance_namespace::LanceNamespace>)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -2027,4 +2045,63 @@ mod tests {
|
||||
let db_options = ListingDatabaseOptions::parse_from_map(&options).unwrap();
|
||||
assert_eq!(db_options.new_table_config.enable_stable_row_ids, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_client() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
|
||||
// Create some tables first
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int32, false),
|
||||
Field::new("name", DataType::Utf8, false),
|
||||
]));
|
||||
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "table1".to_string(),
|
||||
namespace: vec![],
|
||||
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema.clone())),
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "table2".to_string(),
|
||||
namespace: vec![],
|
||||
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Get the namespace client
|
||||
let namespace_client = db.namespace_client().await;
|
||||
assert!(namespace_client.is_ok());
|
||||
let namespace_client = namespace_client.unwrap();
|
||||
|
||||
// Verify the namespace client can list the tables we created
|
||||
// Use empty vec for root namespace
|
||||
let list_result = namespace_client
|
||||
.list_tables(lance_namespace::models::ListTablesRequest {
|
||||
id: Some(vec![]),
|
||||
..Default::default()
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
list_result.is_ok(),
|
||||
"list_tables failed: {:?}",
|
||||
list_result.err()
|
||||
);
|
||||
|
||||
let tables = list_result.unwrap().tables;
|
||||
assert_eq!(tables.len(), 2);
|
||||
assert!(tables.contains(&"table1".to_string()));
|
||||
assert!(tables.contains(&"table2".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -425,6 +425,10 @@ impl Database for LanceNamespaceDatabase {
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
async fn namespace_client(&self) -> Result<Arc<dyn LanceNamespace>> {
|
||||
Ok(self.namespace.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -232,6 +232,38 @@ impl HttpSend for Sender {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed components from a database URL (db://...)
|
||||
pub struct ParsedDbUrl {
|
||||
pub db_name: String,
|
||||
pub db_prefix: Option<String>,
|
||||
}
|
||||
|
||||
/// Parse a database URL and extract the database name and optional prefix.
|
||||
///
|
||||
/// Expected format: `db://db_name` or `db://db_name/prefix`
|
||||
pub fn parse_db_url(db_url: &str) -> Result<ParsedDbUrl> {
|
||||
let parsed_url = url::Url::parse(db_url).map_err(|err| Error::InvalidInput {
|
||||
message: format!("db_url is not a valid URL. '{db_url}'. Error: {err}"),
|
||||
})?;
|
||||
debug_assert_eq!(parsed_url.scheme(), "db");
|
||||
if !parsed_url.has_host() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!("Invalid database URL (missing host) '{}'", db_url),
|
||||
});
|
||||
}
|
||||
let db_name = parsed_url.host_str().unwrap().to_string();
|
||||
let db_prefix = {
|
||||
let prefix = parsed_url.path().trim_start_matches('/');
|
||||
if prefix.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(prefix.to_string())
|
||||
}
|
||||
};
|
||||
|
||||
Ok(ParsedDbUrl { db_name, db_prefix })
|
||||
}
|
||||
|
||||
impl RestfulLanceDbClient<Sender> {
|
||||
fn get_timeout(passed: Option<Duration>, env_var: &str) -> Result<Option<Duration>> {
|
||||
if let Some(passed) = passed {
|
||||
@@ -250,32 +282,12 @@ impl RestfulLanceDbClient<Sender> {
|
||||
}
|
||||
|
||||
pub fn try_new(
|
||||
db_url: &str,
|
||||
api_key: &str,
|
||||
parsed_url: &ParsedDbUrl,
|
||||
region: &str,
|
||||
host_override: Option<String>,
|
||||
default_headers: HeaderMap,
|
||||
client_config: ClientConfig,
|
||||
options: &RemoteOptions,
|
||||
) -> Result<Self> {
|
||||
let parsed_url = url::Url::parse(db_url).map_err(|err| Error::InvalidInput {
|
||||
message: format!("db_url is not a valid URL. '{db_url}'. Error: {err}"),
|
||||
})?;
|
||||
debug_assert_eq!(parsed_url.scheme(), "db");
|
||||
if !parsed_url.has_host() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!("Invalid database URL (missing host) '{}'", db_url),
|
||||
});
|
||||
}
|
||||
let db_name = parsed_url.host_str().unwrap();
|
||||
let db_prefix = {
|
||||
let prefix = parsed_url.path().trim_start_matches('/');
|
||||
if prefix.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(prefix)
|
||||
}
|
||||
};
|
||||
|
||||
// Get the timeouts
|
||||
let timeout =
|
||||
Self::get_timeout(client_config.timeout_config.timeout, "LANCE_CLIENT_TIMEOUT")?;
|
||||
@@ -348,15 +360,7 @@ impl RestfulLanceDbClient<Sender> {
|
||||
}
|
||||
|
||||
let client = client_builder
|
||||
.default_headers(Self::default_headers(
|
||||
api_key,
|
||||
region,
|
||||
db_name,
|
||||
host_override.is_some(),
|
||||
options,
|
||||
db_prefix,
|
||||
&client_config,
|
||||
)?)
|
||||
.default_headers(default_headers)
|
||||
.user_agent(client_config.user_agent)
|
||||
.build()
|
||||
.map_err(|err| Error::Other {
|
||||
@@ -366,7 +370,7 @@ impl RestfulLanceDbClient<Sender> {
|
||||
|
||||
let host = match host_override {
|
||||
Some(host_override) => host_override,
|
||||
None => format!("https://{}.{}.api.lancedb.com", db_name, region),
|
||||
None => format!("https://{}.{}.api.lancedb.com", parsed_url.db_name, region),
|
||||
};
|
||||
debug!("Created client for host: {}", host);
|
||||
let retry_config = client_config.retry_config.clone().try_into()?;
|
||||
@@ -389,7 +393,7 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
|
||||
&self.host
|
||||
}
|
||||
|
||||
fn default_headers(
|
||||
pub fn default_headers(
|
||||
api_key: &str,
|
||||
region: &str,
|
||||
db_name: &str,
|
||||
|
||||
@@ -189,6 +189,10 @@ pub struct RemoteDatabase<S: HttpSend = Sender> {
|
||||
client: RestfulLanceDbClient<S>,
|
||||
table_cache: Cache<String, Arc<RemoteTable<S>>>,
|
||||
uri: String,
|
||||
/// Headers to pass to the namespace client for authentication
|
||||
namespace_headers: HashMap<String, String>,
|
||||
/// TLS configuration for mTLS support
|
||||
tls_config: Option<super::client::TlsConfig>,
|
||||
}
|
||||
|
||||
impl RemoteDatabase {
|
||||
@@ -200,13 +204,32 @@ impl RemoteDatabase {
|
||||
client_config: ClientConfig,
|
||||
options: RemoteOptions,
|
||||
) -> Result<Self> {
|
||||
let client = RestfulLanceDbClient::try_new(
|
||||
uri,
|
||||
let parsed = super::client::parse_db_url(uri)?;
|
||||
let header_map = RestfulLanceDbClient::<Sender>::default_headers(
|
||||
api_key,
|
||||
region,
|
||||
host_override,
|
||||
client_config,
|
||||
&parsed.db_name,
|
||||
host_override.is_some(),
|
||||
&options,
|
||||
parsed.db_prefix.as_deref(),
|
||||
&client_config,
|
||||
)?;
|
||||
|
||||
let namespace_headers: HashMap<String, String> = header_map
|
||||
.iter()
|
||||
.filter_map(|(k, v)| {
|
||||
v.to_str()
|
||||
.ok()
|
||||
.map(|val| (k.as_str().to_string(), val.to_string()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let client = RestfulLanceDbClient::try_new(
|
||||
&parsed,
|
||||
region,
|
||||
host_override,
|
||||
header_map,
|
||||
client_config.clone(),
|
||||
)?;
|
||||
|
||||
let table_cache = Cache::builder()
|
||||
@@ -218,6 +241,8 @@ impl RemoteDatabase {
|
||||
client,
|
||||
table_cache,
|
||||
uri: uri.to_owned(),
|
||||
namespace_headers,
|
||||
tls_config: client_config.tls_config,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -240,6 +265,8 @@ mod test_utils {
|
||||
client,
|
||||
table_cache: Cache::new(0),
|
||||
uri: "http://localhost".to_string(),
|
||||
namespace_headers: HashMap::new(),
|
||||
tls_config: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,11 +275,13 @@ mod test_utils {
|
||||
F: Fn(reqwest::Request) -> http::Response<T> + Send + Sync + 'static,
|
||||
T: Into<reqwest::Body>,
|
||||
{
|
||||
let client = client_with_handler_and_config(handler, config);
|
||||
let client = client_with_handler_and_config(handler, config.clone());
|
||||
Self {
|
||||
client,
|
||||
table_cache: Cache::new(0),
|
||||
uri: "http://localhost".to_string(),
|
||||
namespace_headers: config.extra_headers.clone(),
|
||||
tls_config: config.tls_config.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -716,7 +745,8 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter);
|
||||
let req = self
|
||||
.client
|
||||
.get(&format!("/v1/namespace/{}/describe", namespace_id));
|
||||
.post(&format!("/v1/namespace/{}/describe", namespace_id))
|
||||
.json(&DescribeNamespaceRequest::default());
|
||||
|
||||
let (request_id, resp) = self.client.send(req).await?;
|
||||
let resp = self.client.check_response(&request_id, resp).await?;
|
||||
@@ -727,6 +757,31 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
||||
async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> {
|
||||
// Create a RestNamespace pointing to the same remote host with the same authentication headers
|
||||
let mut builder = lance_namespace_impls::RestNamespaceBuilder::new(self.client.host())
|
||||
.delimiter(&self.client.id_delimiter)
|
||||
// TODO: support header provider
|
||||
.headers(self.namespace_headers.clone());
|
||||
|
||||
// Apply mTLS configuration if present
|
||||
if let Some(tls_config) = &self.tls_config {
|
||||
if let Some(cert_file) = &tls_config.cert_file {
|
||||
builder = builder.cert_file(cert_file);
|
||||
}
|
||||
if let Some(key_file) = &tls_config.key_file {
|
||||
builder = builder.key_file(key_file);
|
||||
}
|
||||
if let Some(ssl_ca_cert) = &tls_config.ssl_ca_cert {
|
||||
builder = builder.ssl_ca_cert(ssl_ca_cert);
|
||||
}
|
||||
builder = builder.assert_hostname(tls_config.assert_hostname);
|
||||
}
|
||||
|
||||
let namespace = builder.build();
|
||||
Ok(Arc::new(namespace) as Arc<dyn lance_namespace::LanceNamespace>)
|
||||
}
|
||||
}
|
||||
|
||||
/// RemoteOptions contains a subset of StorageOptions that are compatible with Remote LanceDB connections
|
||||
@@ -1518,4 +1573,265 @@ mod tests {
|
||||
panic!("Expected HTTP error");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_client() {
|
||||
let conn = Connection::new_with_handler(|_| {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"tables": []}"#)
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
// Get the namespace client from the connection's internal database
|
||||
let namespace_client = conn.namespace_client().await;
|
||||
assert!(namespace_client.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_client_with_tls_config() {
|
||||
use crate::remote::client::TlsConfig;
|
||||
|
||||
let tls_config = TlsConfig {
|
||||
cert_file: Some("/path/to/cert.pem".to_string()),
|
||||
key_file: Some("/path/to/key.pem".to_string()),
|
||||
ssl_ca_cert: Some("/path/to/ca.pem".to_string()),
|
||||
assert_hostname: true,
|
||||
};
|
||||
|
||||
let client_config = ClientConfig {
|
||||
tls_config: Some(tls_config),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let conn = Connection::new_with_handler_and_config(
|
||||
|_| {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"tables": []}"#)
|
||||
.unwrap()
|
||||
},
|
||||
client_config,
|
||||
);
|
||||
|
||||
// Get the namespace client - it should be created with the TLS config
|
||||
let namespace_client = conn.namespace_client().await;
|
||||
assert!(namespace_client.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_namespace_client_with_headers() {
|
||||
let mut extra_headers = HashMap::new();
|
||||
extra_headers.insert("X-Custom-Header".to_string(), "custom-value".to_string());
|
||||
|
||||
let client_config = ClientConfig {
|
||||
extra_headers,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let conn = Connection::new_with_handler_and_config(
|
||||
|_| {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"tables": []}"#)
|
||||
.unwrap()
|
||||
},
|
||||
client_config,
|
||||
);
|
||||
|
||||
// Get the namespace client - it should be created with the extra headers
|
||||
let namespace_client = conn.namespace_client().await;
|
||||
assert!(namespace_client.is_ok());
|
||||
}
|
||||
|
||||
/// Integration tests using RestAdapter to run RemoteDatabase against a real namespace server
|
||||
mod rest_adapter_integration {
|
||||
use super::*;
|
||||
use lance_namespace::models::ListTablesRequest;
|
||||
use lance_namespace_impls::{DirectoryNamespaceBuilder, RestAdapter, RestAdapterConfig};
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Test fixture that manages a REST server backed by DirectoryNamespace
|
||||
struct RestServerFixture {
|
||||
_temp_dir: TempDir,
|
||||
server_handle: lance_namespace_impls::RestAdapterHandle,
|
||||
server_url: String,
|
||||
}
|
||||
|
||||
impl RestServerFixture {
|
||||
async fn new() -> Self {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let temp_path = temp_dir.path().to_str().unwrap().to_string();
|
||||
|
||||
// Create DirectoryNamespace backend
|
||||
let backend = DirectoryNamespaceBuilder::new(&temp_path)
|
||||
.build()
|
||||
.await
|
||||
.unwrap();
|
||||
let backend = Arc::new(backend);
|
||||
|
||||
// Start REST server with port 0 (OS assigns available port)
|
||||
let config = RestAdapterConfig {
|
||||
port: 0,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let server = RestAdapter::new(backend, config);
|
||||
let server_handle = server.start().await.unwrap();
|
||||
|
||||
// Get the actual port assigned by OS
|
||||
let actual_port = server_handle.port();
|
||||
let server_url = format!("http://127.0.0.1:{}", actual_port);
|
||||
|
||||
Self {
|
||||
_temp_dir: temp_dir,
|
||||
server_handle,
|
||||
server_url,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RestServerFixture {
|
||||
fn drop(&mut self) {
|
||||
self.server_handle.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn test_remote_database_with_rest_adapter() {
|
||||
use lance_namespace::models::CreateNamespaceRequest;
|
||||
|
||||
let fixture = RestServerFixture::new().await;
|
||||
|
||||
// Connect to the REST server using lancedb Connection
|
||||
// Use db://dummy as URI and set actual server URL via host_override
|
||||
let conn = ConnectBuilder::new("db://dummy")
|
||||
.api_key("test-api-key")
|
||||
.region("us-east-1")
|
||||
.host_override(&fixture.server_url)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Create a child namespace first
|
||||
let namespace = vec!["test_ns".to_string()];
|
||||
conn.create_namespace(CreateNamespaceRequest {
|
||||
id: Some(namespace.clone()),
|
||||
mode: None,
|
||||
properties: None,
|
||||
})
|
||||
.await
|
||||
.expect("Failed to create namespace");
|
||||
|
||||
// Create a table in the child namespace
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]));
|
||||
let data = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
|
||||
)
|
||||
.unwrap();
|
||||
let reader = RecordBatchIterator::new([Ok(data.clone())], schema.clone());
|
||||
|
||||
let table = conn
|
||||
.create_table("test_table", reader)
|
||||
.namespace(namespace.clone())
|
||||
.execute()
|
||||
.await;
|
||||
assert!(table.is_ok(), "Failed to create table: {:?}", table.err());
|
||||
|
||||
// List tables in the child namespace
|
||||
let list_response = conn
|
||||
.list_tables(ListTablesRequest {
|
||||
id: Some(namespace.clone()),
|
||||
page_token: None,
|
||||
limit: None,
|
||||
})
|
||||
.await
|
||||
.expect("Failed to list tables");
|
||||
assert_eq!(list_response.tables, vec!["test_table"]);
|
||||
|
||||
// Get namespace client and verify it can also list tables
|
||||
let namespace_client = conn.namespace_client().await.unwrap();
|
||||
let list_response = namespace_client
|
||||
.list_tables(ListTablesRequest {
|
||||
id: Some(namespace.clone()),
|
||||
page_token: None,
|
||||
limit: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(list_response.tables, vec!["test_table"]);
|
||||
|
||||
// Open the table from the child namespace
|
||||
let opened_table = conn
|
||||
.open_table("test_table")
|
||||
.namespace(namespace.clone())
|
||||
.execute()
|
||||
.await;
|
||||
assert!(
|
||||
opened_table.is_ok(),
|
||||
"Failed to open table: {:?}",
|
||||
opened_table.err()
|
||||
);
|
||||
assert_eq!(opened_table.unwrap().name(), "test_table");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn test_remote_database_with_multiple_tables() {
|
||||
use lance_namespace::models::CreateNamespaceRequest;
|
||||
|
||||
let fixture = RestServerFixture::new().await;
|
||||
|
||||
// Connect to the REST server
|
||||
// Use db://dummy as URI and set actual server URL via host_override
|
||||
let conn = ConnectBuilder::new("db://dummy")
|
||||
.api_key("test-api-key")
|
||||
.region("us-east-1")
|
||||
.host_override(&fixture.server_url)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Create a child namespace first
|
||||
let namespace = vec!["multi_table_ns".to_string()];
|
||||
conn.create_namespace(CreateNamespaceRequest {
|
||||
id: Some(namespace.clone()),
|
||||
mode: None,
|
||||
properties: None,
|
||||
})
|
||||
.await
|
||||
.expect("Failed to create namespace");
|
||||
|
||||
// Create multiple tables in the child namespace
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
|
||||
for i in 1..=3 {
|
||||
let data =
|
||||
RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![i]))])
|
||||
.unwrap();
|
||||
let reader = RecordBatchIterator::new([Ok(data.clone())], schema.clone());
|
||||
|
||||
conn.create_table(format!("table{}", i), reader)
|
||||
.namespace(namespace.clone())
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("Failed to create table{}: {:?}", i, e));
|
||||
}
|
||||
|
||||
// List tables in the child namespace
|
||||
let list_response = conn
|
||||
.list_tables(ListTablesRequest {
|
||||
id: Some(namespace.clone()),
|
||||
page_token: None,
|
||||
limit: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(list_response.tables.len(), 3);
|
||||
assert!(list_response.tables.contains(&"table1".to_string()));
|
||||
assert!(list_response.tables.contains(&"table2".to_string()));
|
||||
assert!(list_response.tables.contains(&"table3".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user