mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-26 22:59:57 +00:00
Compare commits
13 Commits
python-v0.
...
python-v0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
222e3264ab | ||
|
|
13505026cb | ||
|
|
b0800b4b71 | ||
|
|
1befebf614 | ||
|
|
1ab60fae7f | ||
|
|
e921c90c1b | ||
|
|
05a4ea646a | ||
|
|
ebbeeff4e0 | ||
|
|
407ca53f92 | ||
|
|
ff71d7e552 | ||
|
|
2261eb95a0 | ||
|
|
5b397e410b | ||
|
|
b5a39bffec |
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.22.1-beta.1"
|
current_version = "0.22.1-beta.3"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
4
.github/workflows/pypi-publish.yml
vendored
4
.github/workflows/pypi-publish.yml
vendored
@@ -56,7 +56,7 @@ jobs:
|
|||||||
pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
|
pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
fury_token: ${{ secrets.FURY_TOKEN }}
|
||||||
mac:
|
mac:
|
||||||
timeout-minutes: 60
|
timeout-minutes: 90
|
||||||
runs-on: ${{ matrix.config.runner }}
|
runs-on: ${{ matrix.config.runner }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -64,7 +64,7 @@ jobs:
|
|||||||
- target: x86_64-apple-darwin
|
- target: x86_64-apple-darwin
|
||||||
runner: macos-13
|
runner: macos-13
|
||||||
- target: aarch64-apple-darwin
|
- target: aarch64-apple-darwin
|
||||||
runner: macos-14
|
runner: warp-macos-14-arm64-6x
|
||||||
env:
|
env:
|
||||||
MACOSX_DEPLOYMENT_TARGET: 10.15
|
MACOSX_DEPLOYMENT_TARGET: 10.15
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
106
Cargo.lock
generated
106
Cargo.lock
generated
@@ -1153,7 +1153,7 @@ dependencies = [
|
|||||||
"bitflags 2.9.4",
|
"bitflags 2.9.4",
|
||||||
"cexpr",
|
"cexpr",
|
||||||
"clang-sys",
|
"clang-sys",
|
||||||
"itertools 0.11.0",
|
"itertools 0.12.1",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"lazycell",
|
"lazycell",
|
||||||
"log",
|
"log",
|
||||||
@@ -2929,6 +2929,18 @@ version = "0.2.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
|
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fastbloom"
|
||||||
|
version = "0.14.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "18c1ddb9231d8554c2d6bdf4cfaabf0c59251658c68b6c95cd52dd0c513a912a"
|
||||||
|
dependencies = [
|
||||||
|
"getrandom 0.3.3",
|
||||||
|
"libm",
|
||||||
|
"rand 0.9.2",
|
||||||
|
"siphasher",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fastdivide"
|
name = "fastdivide"
|
||||||
version = "0.4.2"
|
version = "0.4.2"
|
||||||
@@ -3028,8 +3040,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fsst"
|
name = "fsst"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fe0a0b1d16ce6b863be8ab766004d89ebf0779fd6ce31b0ef3bbc7fedaaad373"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"rand 0.9.2",
|
"rand 0.9.2",
|
||||||
@@ -4206,8 +4219,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance"
|
name = "lance"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "42171f2af5d377e6bbcc8a8572144ee15b73a8f78ceb6160f1adeabf0d0f3e3c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4270,8 +4284,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-arrow"
|
name = "lance-arrow"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "25ef9499a1e581112f45fbf743fdc8e24830cda0bd13396b11c71aa6e6cba083"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4289,8 +4304,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-bitpacking"
|
name = "lance-bitpacking"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1101fffd5b161bbdc6e932d6c0a7f94cb1752b0f8cd6d18ef9064052ab901a84"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrayref",
|
"arrayref",
|
||||||
"paste",
|
"paste",
|
||||||
@@ -4299,8 +4315,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-core"
|
name = "lance-core"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "527ee5e6472d058d8c66c702fbe318a3f60f971e652e60dcfc6349bdbc9b0733"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4335,8 +4352,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datafusion"
|
name = "lance-datafusion"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "65a80f7f15f2d941ec7b8253625cbb8e12081ea27584dd1fbc657fb9fb377f7a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4365,8 +4383,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datagen"
|
name = "lance-datagen"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0495c8afa18f246ac4b337c47d7827560283783963dd2177862d91161478fd79"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4383,8 +4402,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-encoding"
|
name = "lance-encoding"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0e80e9ae49d68b95d58e77d9177f68983dce4f0803ef42840e1631b38dd66adc"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4412,6 +4432,7 @@ dependencies = [
|
|||||||
"prost-types",
|
"prost-types",
|
||||||
"rand 0.9.2",
|
"rand 0.9.2",
|
||||||
"snafu",
|
"snafu",
|
||||||
|
"strum",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
"xxhash-rust",
|
"xxhash-rust",
|
||||||
@@ -4420,8 +4441,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-file"
|
name = "lance-file"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f1707f9f5097b36c82d3a8524bb41c762c80d5dfa5e32aa7bfc6a1c0847a1cce"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4455,8 +4477,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-index"
|
name = "lance-index"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "28ab52586a5a7f5371a5abf4862968231f8c0232ce0780bc456f1ec16e9370f9"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4477,6 +4500,7 @@ dependencies = [
|
|||||||
"datafusion-sql",
|
"datafusion-sql",
|
||||||
"deepsize",
|
"deepsize",
|
||||||
"dirs",
|
"dirs",
|
||||||
|
"fastbloom",
|
||||||
"fst",
|
"fst",
|
||||||
"futures",
|
"futures",
|
||||||
"half",
|
"half",
|
||||||
@@ -4491,6 +4515,7 @@ dependencies = [
|
|||||||
"lance-io",
|
"lance-io",
|
||||||
"lance-linalg",
|
"lance-linalg",
|
||||||
"lance-table",
|
"lance-table",
|
||||||
|
"libm",
|
||||||
"log",
|
"log",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"object_store",
|
"object_store",
|
||||||
@@ -4507,13 +4532,15 @@ dependencies = [
|
|||||||
"tempfile",
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
"twox-hash",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-io"
|
name = "lance-io"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d606f9f6a7f8ec2cacf28dfce7b2fc39e7db9f0ec77f907b8e47c756e3dd163b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4553,8 +4580,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-linalg"
|
name = "lance-linalg"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c9f1a94a5d966ff1eae817a835e3a57b34f73300f83a43bb28e7e2806695b8ba"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4577,8 +4605,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-table"
|
name = "lance-table"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "fac5c0ca6e5c285645465b95fb99fc464a1fd22a6d4b32ae0e0760f06b4b8a7f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4616,8 +4645,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-testing"
|
name = "lance-testing"
|
||||||
version = "0.35.0"
|
version = "0.37.0"
|
||||||
source = "git+https://github.com/lancedb/lance.git?tag=v0.35.0-beta.4#e842a8f922b90c298c356dd1c6afdc83ca5253f2"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "384acc1dd13379a2ae24f3e3635d9c1f4fb4dc1534f7ffd2740c268f2eb73455"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-schema",
|
"arrow-schema",
|
||||||
@@ -4628,7 +4658,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.22.1-beta.1"
|
version = "0.22.1-beta.3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4715,7 +4745,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
version = "0.22.1-beta.1"
|
version = "0.22.1-beta.3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-ipc",
|
"arrow-ipc",
|
||||||
@@ -4735,7 +4765,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.25.1-beta.1"
|
version = "0.25.1-beta.3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -7781,6 +7811,15 @@ version = "0.11.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "strum"
|
||||||
|
version = "0.25.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
|
||||||
|
dependencies = [
|
||||||
|
"strum_macros",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "strum_macros"
|
name = "strum_macros"
|
||||||
version = "0.25.3"
|
version = "0.25.3"
|
||||||
@@ -8441,6 +8480,9 @@ name = "twox-hash"
|
|||||||
version = "2.1.1"
|
version = "2.1.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56"
|
checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56"
|
||||||
|
dependencies = [
|
||||||
|
"rand 0.9.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "typenum"
|
name = "typenum"
|
||||||
|
|||||||
16
Cargo.toml
16
Cargo.toml
@@ -15,14 +15,14 @@ categories = ["database-implementations"]
|
|||||||
rust-version = "1.78.0"
|
rust-version = "1.78.0"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
lance = { "version" = "=0.35.0", default-features = false, "features" = ["dynamodb"], "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance = { "version" = "=0.37.0", default-features = false, "features" = ["dynamodb"] }
|
||||||
lance-io = { "version" = "=0.35.0", default-features = false, "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-io = { "version" = "=0.37.0", default-features = false }
|
||||||
lance-index = { "version" = "=0.35.0", "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-index = "=0.37.0"
|
||||||
lance-linalg = { "version" = "=0.35.0", "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-linalg = "=0.37.0"
|
||||||
lance-table = { "version" = "=0.35.0", "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-table = "=0.37.0"
|
||||||
lance-testing = { "version" = "=0.35.0", "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-testing = "=0.37.0"
|
||||||
lance-datafusion = { "version" = "=0.35.0", "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-datafusion = "=0.37.0"
|
||||||
lance-encoding = { "version" = "=0.35.0", "tag" = "v0.35.0-beta.4", "git" = "https://github.com/lancedb/lance.git" }
|
lance-encoding = "=0.37.0"
|
||||||
# Note that this one does not include pyarrow
|
# Note that this one does not include pyarrow
|
||||||
arrow = { version = "55.1", optional = false }
|
arrow = { version = "55.1", optional = false }
|
||||||
arrow-array = "55.1"
|
arrow-array = "55.1"
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import argparse
|
import argparse
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
import json
|
import json
|
||||||
|
|
||||||
@@ -18,8 +19,12 @@ def run_command(command: str) -> str:
|
|||||||
|
|
||||||
def get_latest_stable_version() -> str:
|
def get_latest_stable_version() -> str:
|
||||||
version_line = run_command("cargo info lance | grep '^version:'")
|
version_line = run_command("cargo info lance | grep '^version:'")
|
||||||
version = version_line.split(" ")[1].strip()
|
# Example output: "version: 0.35.0 (latest 0.37.0)"
|
||||||
return version
|
match = re.search(r'\(latest ([0-9.]+)\)', version_line)
|
||||||
|
if match:
|
||||||
|
return match.group(1)
|
||||||
|
# Fallback: use the first version after 'version:'
|
||||||
|
return version_line.split("version:")[1].split()[0].strip()
|
||||||
|
|
||||||
|
|
||||||
def get_latest_preview_version() -> str:
|
def get_latest_preview_version() -> str:
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.22.1-beta.1</version>
|
<version>0.22.1-beta.3</version>
|
||||||
<relativePath>../pom.xml</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.22.1-beta.1</version>
|
<version>0.22.1-beta.3</version>
|
||||||
<relativePath>../pom.xml</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.22.1-beta.1</version>
|
<version>0.22.1-beta.3</version>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<name>${project.artifactId}</name>
|
<name>${project.artifactId}</name>
|
||||||
<description>LanceDB Java SDK Parent POM</description>
|
<description>LanceDB Java SDK Parent POM</description>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
version = "0.22.1-beta.1"
|
version = "0.22.1-beta.3"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description.workspace = true
|
description.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
|
|||||||
@@ -1008,5 +1008,64 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
|||||||
expect(result).toEqual(null);
|
expect(result).toEqual(null);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("boolean null handling", function () {
|
||||||
|
it("should handle null values in nullable boolean fields", () => {
|
||||||
|
const { makeArrowTable } = require("../lancedb/arrow");
|
||||||
|
const schema = new Schema([new Field("test", new arrow.Bool(), true)]);
|
||||||
|
|
||||||
|
// Test with all null values
|
||||||
|
const data = [{ test: null }];
|
||||||
|
const table = makeArrowTable(data, { schema });
|
||||||
|
|
||||||
|
expect(table.numRows).toBe(1);
|
||||||
|
expect(table.schema.names).toEqual(["test"]);
|
||||||
|
expect(table.getChild("test")!.get(0)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should handle mixed null and non-null boolean values", () => {
|
||||||
|
const { makeArrowTable } = require("../lancedb/arrow");
|
||||||
|
const schema = new Schema([new Field("test", new Bool(), true)]);
|
||||||
|
|
||||||
|
// Test with mixed values
|
||||||
|
const data = [{ test: true }, { test: null }, { test: false }];
|
||||||
|
const table = makeArrowTable(data, { schema });
|
||||||
|
|
||||||
|
expect(table.numRows).toBe(3);
|
||||||
|
expect(table.getChild("test")!.get(0)).toBe(true);
|
||||||
|
expect(table.getChild("test")!.get(1)).toBeNull();
|
||||||
|
expect(table.getChild("test")!.get(2)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Test for the undefined values bug fix
|
||||||
|
describe("undefined values handling", () => {
|
||||||
|
it("should handle mixed undefined and actual values", () => {
|
||||||
|
const schema = new Schema([
|
||||||
|
new Field("text", new Utf8(), true), // nullable
|
||||||
|
new Field("number", new Int32(), true), // nullable
|
||||||
|
new Field("bool", new Bool(), true), // nullable
|
||||||
|
]);
|
||||||
|
|
||||||
|
const data = [
|
||||||
|
{ text: undefined, number: 42, bool: true },
|
||||||
|
{ text: "hello", number: undefined, bool: false },
|
||||||
|
{ text: "world", number: 123, bool: undefined },
|
||||||
|
];
|
||||||
|
const table = makeArrowTable(data, { schema });
|
||||||
|
|
||||||
|
const result = table.toArray();
|
||||||
|
expect(result).toHaveLength(3);
|
||||||
|
expect(result[0].text).toBe(null);
|
||||||
|
expect(result[0].number).toBe(42);
|
||||||
|
expect(result[0].bool).toBe(true);
|
||||||
|
expect(result[1].text).toBe("hello");
|
||||||
|
expect(result[1].number).toBe(null);
|
||||||
|
expect(result[1].bool).toBe(false);
|
||||||
|
expect(result[2].text).toBe("world");
|
||||||
|
expect(result[2].number).toBe(123);
|
||||||
|
expect(result[2].bool).toBe(null);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -203,3 +203,106 @@ describe("given a connection", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("clone table functionality", () => {
|
||||||
|
let tmpDir: tmp.DirResult;
|
||||||
|
let db: Connection;
|
||||||
|
beforeEach(async () => {
|
||||||
|
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||||
|
db = await connect(tmpDir.name);
|
||||||
|
});
|
||||||
|
afterEach(() => tmpDir.removeCallback());
|
||||||
|
|
||||||
|
it("should clone a table with latest version (default behavior)", async () => {
|
||||||
|
// Create source table with some data
|
||||||
|
const data = [
|
||||||
|
{ id: 1, text: "hello", vector: [1.0, 2.0] },
|
||||||
|
{ id: 2, text: "world", vector: [3.0, 4.0] },
|
||||||
|
];
|
||||||
|
const sourceTable = await db.createTable("source", data);
|
||||||
|
|
||||||
|
// Add more data to create a new version
|
||||||
|
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
|
||||||
|
await sourceTable.add(moreData);
|
||||||
|
|
||||||
|
// Clone the table (should get latest version with 3 rows)
|
||||||
|
const sourceUri = `${tmpDir.name}/source.lance`;
|
||||||
|
const clonedTable = await db.cloneTable("cloned", sourceUri);
|
||||||
|
|
||||||
|
// Verify cloned table has all 3 rows
|
||||||
|
expect(await clonedTable.countRows()).toBe(3);
|
||||||
|
expect((await db.tableNames()).includes("cloned")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should clone a table from a specific version", async () => {
|
||||||
|
// Create source table with initial data
|
||||||
|
const data = [
|
||||||
|
{ id: 1, text: "hello", vector: [1.0, 2.0] },
|
||||||
|
{ id: 2, text: "world", vector: [3.0, 4.0] },
|
||||||
|
];
|
||||||
|
const sourceTable = await db.createTable("source", data);
|
||||||
|
|
||||||
|
// Get the initial version
|
||||||
|
const initialVersion = await sourceTable.version();
|
||||||
|
|
||||||
|
// Add more data to create a new version
|
||||||
|
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
|
||||||
|
await sourceTable.add(moreData);
|
||||||
|
|
||||||
|
// Verify source now has 3 rows
|
||||||
|
expect(await sourceTable.countRows()).toBe(3);
|
||||||
|
|
||||||
|
// Clone from the initial version (should have only 2 rows)
|
||||||
|
const sourceUri = `${tmpDir.name}/source.lance`;
|
||||||
|
const clonedTable = await db.cloneTable("cloned", sourceUri, {
|
||||||
|
sourceVersion: initialVersion,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Verify cloned table has only the initial 2 rows
|
||||||
|
expect(await clonedTable.countRows()).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should clone a table from a tagged version", async () => {
|
||||||
|
// Create source table with initial data
|
||||||
|
const data = [
|
||||||
|
{ id: 1, text: "hello", vector: [1.0, 2.0] },
|
||||||
|
{ id: 2, text: "world", vector: [3.0, 4.0] },
|
||||||
|
];
|
||||||
|
const sourceTable = await db.createTable("source", data);
|
||||||
|
|
||||||
|
// Create a tag for the current version
|
||||||
|
const tags = await sourceTable.tags();
|
||||||
|
await tags.create("v1.0", await sourceTable.version());
|
||||||
|
|
||||||
|
// Add more data after the tag
|
||||||
|
const moreData = [{ id: 3, text: "test", vector: [5.0, 6.0] }];
|
||||||
|
await sourceTable.add(moreData);
|
||||||
|
|
||||||
|
// Verify source now has 3 rows
|
||||||
|
expect(await sourceTable.countRows()).toBe(3);
|
||||||
|
|
||||||
|
// Clone from the tagged version (should have only 2 rows)
|
||||||
|
const sourceUri = `${tmpDir.name}/source.lance`;
|
||||||
|
const clonedTable = await db.cloneTable("cloned", sourceUri, {
|
||||||
|
sourceTag: "v1.0",
|
||||||
|
});
|
||||||
|
|
||||||
|
// Verify cloned table has only the tagged version's 2 rows
|
||||||
|
expect(await clonedTable.countRows()).toBe(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should fail when attempting deep clone", async () => {
|
||||||
|
// Create source table with some data
|
||||||
|
const data = [
|
||||||
|
{ id: 1, text: "hello", vector: [1.0, 2.0] },
|
||||||
|
{ id: 2, text: "world", vector: [3.0, 4.0] },
|
||||||
|
];
|
||||||
|
await db.createTable("source", data);
|
||||||
|
|
||||||
|
// Try to create a deep clone (should fail)
|
||||||
|
const sourceUri = `${tmpDir.name}/source.lance`;
|
||||||
|
await expect(
|
||||||
|
db.cloneTable("cloned", sourceUri, { isShallow: false }),
|
||||||
|
).rejects.toThrow("Deep clone is not yet implemented");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -256,6 +256,60 @@ describe("embedding functions", () => {
|
|||||||
expect(actual).toHaveProperty("text");
|
expect(actual).toHaveProperty("text");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("should handle undefined vector field with embedding function correctly", async () => {
|
||||||
|
@register("undefined_test")
|
||||||
|
class MockEmbeddingFunction extends EmbeddingFunction<string> {
|
||||||
|
ndims() {
|
||||||
|
return 3;
|
||||||
|
}
|
||||||
|
embeddingDataType(): Float {
|
||||||
|
return new Float32();
|
||||||
|
}
|
||||||
|
async computeQueryEmbeddings(_data: string) {
|
||||||
|
return [1, 2, 3];
|
||||||
|
}
|
||||||
|
async computeSourceEmbeddings(data: string[]) {
|
||||||
|
return Array.from({ length: data.length }).fill([
|
||||||
|
1, 2, 3,
|
||||||
|
]) as number[][];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const func = getRegistry()
|
||||||
|
.get<MockEmbeddingFunction>("undefined_test")!
|
||||||
|
.create();
|
||||||
|
const schema = new Schema([
|
||||||
|
new Field("text", new Utf8(), true),
|
||||||
|
new Field(
|
||||||
|
"vector",
|
||||||
|
new FixedSizeList(3, new Field("item", new Float32(), true)),
|
||||||
|
true,
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const db = await connect(tmpDir.name);
|
||||||
|
const table = await db.createEmptyTable("test_undefined", schema, {
|
||||||
|
embeddingFunction: {
|
||||||
|
function: func,
|
||||||
|
sourceColumn: "text",
|
||||||
|
vectorColumn: "vector",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
// Test that undefined, null, and omitted vector fields all work
|
||||||
|
await table.add([{ text: "test1", vector: undefined }]);
|
||||||
|
await table.add([{ text: "test2", vector: null }]);
|
||||||
|
await table.add([{ text: "test3" }]);
|
||||||
|
|
||||||
|
const rows = await table.query().toArray();
|
||||||
|
expect(rows.length).toBe(3);
|
||||||
|
|
||||||
|
// All rows should have vectors computed by the embedding function
|
||||||
|
for (const row of rows) {
|
||||||
|
expect(row.vector).toBeDefined();
|
||||||
|
expect(JSON.parse(JSON.stringify(row.vector))).toEqual([1, 2, 3]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test.each([new Float16(), new Float32(), new Float64()])(
|
test.each([new Float16(), new Float32(), new Float64()])(
|
||||||
"should be able to provide manual embeddings with multiple float datatype",
|
"should be able to provide manual embeddings with multiple float datatype",
|
||||||
async (floatType) => {
|
async (floatType) => {
|
||||||
|
|||||||
@@ -512,7 +512,11 @@ function* rowPathsAndValues(
|
|||||||
if (isObject(value)) {
|
if (isObject(value)) {
|
||||||
yield* rowPathsAndValues(value, [...basePath, key]);
|
yield* rowPathsAndValues(value, [...basePath, key]);
|
||||||
} else {
|
} else {
|
||||||
yield [[...basePath, key], value];
|
// Skip undefined values - they should be treated the same as missing fields
|
||||||
|
// for embedding function purposes
|
||||||
|
if (value !== undefined) {
|
||||||
|
yield [[...basePath, key], value];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -701,7 +705,7 @@ function transposeData(
|
|||||||
}
|
}
|
||||||
return current;
|
return current;
|
||||||
});
|
});
|
||||||
return makeVector(values, field.type);
|
return makeVector(values, field.type, undefined, field.nullable);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -748,9 +752,30 @@ function makeVector(
|
|||||||
values: unknown[],
|
values: unknown[],
|
||||||
type?: DataType,
|
type?: DataType,
|
||||||
stringAsDictionary?: boolean,
|
stringAsDictionary?: boolean,
|
||||||
|
nullable?: boolean,
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: skip
|
// biome-ignore lint/suspicious/noExplicitAny: skip
|
||||||
): Vector<any> {
|
): Vector<any> {
|
||||||
if (type !== undefined) {
|
if (type !== undefined) {
|
||||||
|
// Convert undefined values to null for nullable fields
|
||||||
|
if (nullable) {
|
||||||
|
values = values.map((v) => (v === undefined ? null : v));
|
||||||
|
}
|
||||||
|
|
||||||
|
// workaround for: https://github.com/apache/arrow-js/issues/68
|
||||||
|
if (DataType.isBool(type)) {
|
||||||
|
const hasNonNullValue = values.some((v) => v !== null && v !== undefined);
|
||||||
|
if (!hasNonNullValue) {
|
||||||
|
const nullBitmap = new Uint8Array(Math.ceil(values.length / 8));
|
||||||
|
const data = makeData({
|
||||||
|
type: type,
|
||||||
|
length: values.length,
|
||||||
|
nullCount: values.length,
|
||||||
|
nullBitmap,
|
||||||
|
});
|
||||||
|
return arrowMakeVector(data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// No need for inference, let Arrow create it
|
// No need for inference, let Arrow create it
|
||||||
if (type instanceof Int) {
|
if (type instanceof Int) {
|
||||||
if (DataType.isInt(type) && type.bitWidth === 64) {
|
if (DataType.isInt(type) && type.bitWidth === 64) {
|
||||||
@@ -875,7 +900,12 @@ async function applyEmbeddingsFromMetadata(
|
|||||||
for (const field of schema.fields) {
|
for (const field of schema.fields) {
|
||||||
if (!(field.name in columns)) {
|
if (!(field.name in columns)) {
|
||||||
const nullValues = new Array(table.numRows).fill(null);
|
const nullValues = new Array(table.numRows).fill(null);
|
||||||
columns[field.name] = makeVector(nullValues, field.type);
|
columns[field.name] = makeVector(
|
||||||
|
nullValues,
|
||||||
|
field.type,
|
||||||
|
undefined,
|
||||||
|
field.nullable,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -939,7 +969,12 @@ async function applyEmbeddings<T>(
|
|||||||
} else if (schema != null) {
|
} else if (schema != null) {
|
||||||
const destField = schema.fields.find((f) => f.name === destColumn);
|
const destField = schema.fields.find((f) => f.name === destColumn);
|
||||||
if (destField != null) {
|
if (destField != null) {
|
||||||
newColumns[destColumn] = makeVector([], destField.type);
|
newColumns[destColumn] = makeVector(
|
||||||
|
[],
|
||||||
|
destField.type,
|
||||||
|
undefined,
|
||||||
|
destField.nullable,
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`,
|
`Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`,
|
||||||
|
|||||||
@@ -268,6 +268,33 @@ export abstract class Connection {
|
|||||||
* @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
|
* @param {string[]} namespace The namespace to drop tables from (defaults to root namespace).
|
||||||
*/
|
*/
|
||||||
abstract dropAllTables(namespace?: string[]): Promise<void>;
|
abstract dropAllTables(namespace?: string[]): Promise<void>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clone a table from a source table.
|
||||||
|
*
|
||||||
|
* A shallow clone creates a new table that shares the underlying data files
|
||||||
|
* with the source table but has its own independent manifest. This allows
|
||||||
|
* both the source and cloned tables to evolve independently while initially
|
||||||
|
* sharing the same data, deletion, and index files.
|
||||||
|
*
|
||||||
|
* @param {string} targetTableName - The name of the target table to create.
|
||||||
|
* @param {string} sourceUri - The URI of the source table to clone from.
|
||||||
|
* @param {object} options - Clone options.
|
||||||
|
* @param {string[]} options.targetNamespace - The namespace for the target table (defaults to root namespace).
|
||||||
|
* @param {number} options.sourceVersion - The version of the source table to clone.
|
||||||
|
* @param {string} options.sourceTag - The tag of the source table to clone.
|
||||||
|
* @param {boolean} options.isShallow - Whether to perform a shallow clone (defaults to true).
|
||||||
|
*/
|
||||||
|
abstract cloneTable(
|
||||||
|
targetTableName: string,
|
||||||
|
sourceUri: string,
|
||||||
|
options?: {
|
||||||
|
targetNamespace?: string[];
|
||||||
|
sourceVersion?: number;
|
||||||
|
sourceTag?: string;
|
||||||
|
isShallow?: boolean;
|
||||||
|
},
|
||||||
|
): Promise<Table>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @hideconstructor */
|
/** @hideconstructor */
|
||||||
@@ -332,6 +359,28 @@ export class LocalConnection extends Connection {
|
|||||||
return new LocalTable(innerTable);
|
return new LocalTable(innerTable);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async cloneTable(
|
||||||
|
targetTableName: string,
|
||||||
|
sourceUri: string,
|
||||||
|
options?: {
|
||||||
|
targetNamespace?: string[];
|
||||||
|
sourceVersion?: number;
|
||||||
|
sourceTag?: string;
|
||||||
|
isShallow?: boolean;
|
||||||
|
},
|
||||||
|
): Promise<Table> {
|
||||||
|
const innerTable = await this.inner.cloneTable(
|
||||||
|
targetTableName,
|
||||||
|
sourceUri,
|
||||||
|
options?.targetNamespace ?? [],
|
||||||
|
options?.sourceVersion ?? null,
|
||||||
|
options?.sourceTag ?? null,
|
||||||
|
options?.isShallow ?? true,
|
||||||
|
);
|
||||||
|
|
||||||
|
return new LocalTable(innerTable);
|
||||||
|
}
|
||||||
|
|
||||||
private getStorageOptions(
|
private getStorageOptions(
|
||||||
options?: Partial<CreateTableOptions>,
|
options?: Partial<CreateTableOptions>,
|
||||||
): Record<string, string> | undefined {
|
): Record<string, string> | undefined {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-arm64",
|
"name": "@lancedb/lancedb-darwin-arm64",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.darwin-arm64.node",
|
"main": "lancedb.darwin-arm64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-x64",
|
"name": "@lancedb/lancedb-darwin-x64",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.darwin-x64.node",
|
"main": "lancedb.darwin-x64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-gnu.node",
|
"main": "lancedb.linux-arm64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-musl.node",
|
"main": "lancedb.linux-arm64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-gnu.node",
|
"main": "lancedb.linux-x64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-musl.node",
|
"main": "lancedb.linux-x64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": [
|
"os": [
|
||||||
"win32"
|
"win32"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"os": ["win32"],
|
"os": ["win32"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.win32-x64-msvc.node",
|
"main": "lancedb.win32-x64-msvc.node",
|
||||||
|
|||||||
4
nodejs/package-lock.json
generated
4
nodejs/package-lock.json
generated
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64",
|
"x64",
|
||||||
"arm64"
|
"arm64"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
"ann"
|
"ann"
|
||||||
],
|
],
|
||||||
"private": false,
|
"private": false,
|
||||||
"version": "0.22.1-beta.1",
|
"version": "0.22.1-beta.3",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./dist/index.js",
|
".": "./dist/index.js",
|
||||||
|
|||||||
@@ -213,6 +213,36 @@ impl Connection {
|
|||||||
Ok(Table::new(tbl))
|
Ok(Table::new(tbl))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[napi(catch_unwind)]
|
||||||
|
pub async fn clone_table(
|
||||||
|
&self,
|
||||||
|
target_table_name: String,
|
||||||
|
source_uri: String,
|
||||||
|
target_namespace: Vec<String>,
|
||||||
|
source_version: Option<i64>,
|
||||||
|
source_tag: Option<String>,
|
||||||
|
is_shallow: bool,
|
||||||
|
) -> napi::Result<Table> {
|
||||||
|
let mut builder = self
|
||||||
|
.get_inner()?
|
||||||
|
.clone_table(&target_table_name, &source_uri);
|
||||||
|
|
||||||
|
builder = builder.target_namespace(target_namespace);
|
||||||
|
|
||||||
|
if let Some(version) = source_version {
|
||||||
|
builder = builder.source_version(version as u64);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(tag) = source_tag {
|
||||||
|
builder = builder.source_tag(tag);
|
||||||
|
}
|
||||||
|
|
||||||
|
builder = builder.is_shallow(is_shallow);
|
||||||
|
|
||||||
|
let tbl = builder.execute().await.default_error()?;
|
||||||
|
Ok(Table::new(tbl))
|
||||||
|
}
|
||||||
|
|
||||||
/// Drop table with the name. Or raise an error if the table does not exist.
|
/// Drop table with the name. Or raise an error if the table does not exist.
|
||||||
#[napi(catch_unwind)]
|
#[napi(catch_unwind)]
|
||||||
pub async fn drop_table(&self, name: String, namespace: Vec<String>) -> napi::Result<()> {
|
pub async fn drop_table(&self, name: String, namespace: Vec<String>) -> napi::Result<()> {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.25.1-beta.2"
|
current_version = "0.25.1"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.25.1-beta.2"
|
version = "0.25.1"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -60,6 +60,15 @@ class Connection(object):
|
|||||||
storage_options: Optional[Dict[str, str]] = None,
|
storage_options: Optional[Dict[str, str]] = None,
|
||||||
index_cache_size: Optional[int] = None,
|
index_cache_size: Optional[int] = None,
|
||||||
) -> Table: ...
|
) -> Table: ...
|
||||||
|
async def clone_table(
|
||||||
|
self,
|
||||||
|
target_table_name: str,
|
||||||
|
source_uri: str,
|
||||||
|
target_namespace: List[str] = [],
|
||||||
|
source_version: Optional[int] = None,
|
||||||
|
source_tag: Optional[str] = None,
|
||||||
|
is_shallow: bool = True,
|
||||||
|
) -> Table: ...
|
||||||
async def rename_table(
|
async def rename_table(
|
||||||
self,
|
self,
|
||||||
cur_name: str,
|
cur_name: str,
|
||||||
|
|||||||
@@ -665,6 +665,60 @@ class LanceDBConnection(DBConnection):
|
|||||||
index_cache_size=index_cache_size,
|
index_cache_size=index_cache_size,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def clone_table(
|
||||||
|
self,
|
||||||
|
target_table_name: str,
|
||||||
|
source_uri: str,
|
||||||
|
*,
|
||||||
|
target_namespace: List[str] = [],
|
||||||
|
source_version: Optional[int] = None,
|
||||||
|
source_tag: Optional[str] = None,
|
||||||
|
is_shallow: bool = True,
|
||||||
|
) -> LanceTable:
|
||||||
|
"""Clone a table from a source table.
|
||||||
|
|
||||||
|
A shallow clone creates a new table that shares the underlying data files
|
||||||
|
with the source table but has its own independent manifest. This allows
|
||||||
|
both the source and cloned tables to evolve independently while initially
|
||||||
|
sharing the same data, deletion, and index files.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
target_table_name: str
|
||||||
|
The name of the target table to create.
|
||||||
|
source_uri: str
|
||||||
|
The URI of the source table to clone from.
|
||||||
|
target_namespace: List[str], optional
|
||||||
|
The namespace for the target table.
|
||||||
|
None or empty list represents root namespace.
|
||||||
|
source_version: int, optional
|
||||||
|
The version of the source table to clone.
|
||||||
|
source_tag: str, optional
|
||||||
|
The tag of the source table to clone.
|
||||||
|
is_shallow: bool, default True
|
||||||
|
Whether to perform a shallow clone (True) or deep clone (False).
|
||||||
|
Currently only shallow clone is supported.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
A LanceTable object representing the cloned table.
|
||||||
|
"""
|
||||||
|
LOOP.run(
|
||||||
|
self._conn.clone_table(
|
||||||
|
target_table_name,
|
||||||
|
source_uri,
|
||||||
|
target_namespace=target_namespace,
|
||||||
|
source_version=source_version,
|
||||||
|
source_tag=source_tag,
|
||||||
|
is_shallow=is_shallow,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return LanceTable.open(
|
||||||
|
self,
|
||||||
|
target_table_name,
|
||||||
|
namespace=target_namespace,
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def drop_table(
|
def drop_table(
|
||||||
self,
|
self,
|
||||||
@@ -1136,6 +1190,54 @@ class AsyncConnection(object):
|
|||||||
)
|
)
|
||||||
return AsyncTable(table)
|
return AsyncTable(table)
|
||||||
|
|
||||||
|
async def clone_table(
|
||||||
|
self,
|
||||||
|
target_table_name: str,
|
||||||
|
source_uri: str,
|
||||||
|
*,
|
||||||
|
target_namespace: List[str] = [],
|
||||||
|
source_version: Optional[int] = None,
|
||||||
|
source_tag: Optional[str] = None,
|
||||||
|
is_shallow: bool = True,
|
||||||
|
) -> AsyncTable:
|
||||||
|
"""Clone a table from a source table.
|
||||||
|
|
||||||
|
A shallow clone creates a new table that shares the underlying data files
|
||||||
|
with the source table but has its own independent manifest. This allows
|
||||||
|
both the source and cloned tables to evolve independently while initially
|
||||||
|
sharing the same data, deletion, and index files.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
target_table_name: str
|
||||||
|
The name of the target table to create.
|
||||||
|
source_uri: str
|
||||||
|
The URI of the source table to clone from.
|
||||||
|
target_namespace: List[str], optional
|
||||||
|
The namespace for the target table.
|
||||||
|
None or empty list represents root namespace.
|
||||||
|
source_version: int, optional
|
||||||
|
The version of the source table to clone.
|
||||||
|
source_tag: str, optional
|
||||||
|
The tag of the source table to clone.
|
||||||
|
is_shallow: bool, default True
|
||||||
|
Whether to perform a shallow clone (True) or deep clone (False).
|
||||||
|
Currently only shallow clone is supported.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
An AsyncTable object representing the cloned table.
|
||||||
|
"""
|
||||||
|
table = await self._inner.clone_table(
|
||||||
|
target_table_name,
|
||||||
|
source_uri,
|
||||||
|
target_namespace=target_namespace,
|
||||||
|
source_version=source_version,
|
||||||
|
source_tag=source_tag,
|
||||||
|
is_shallow=is_shallow,
|
||||||
|
)
|
||||||
|
return AsyncTable(table)
|
||||||
|
|
||||||
async def rename_table(
|
async def rename_table(
|
||||||
self,
|
self,
|
||||||
cur_name: str,
|
cur_name: str,
|
||||||
|
|||||||
@@ -212,6 +212,53 @@ class RemoteDBConnection(DBConnection):
|
|||||||
table = LOOP.run(self._conn.open_table(name, namespace=namespace))
|
table = LOOP.run(self._conn.open_table(name, namespace=namespace))
|
||||||
return RemoteTable(table, self.db_name)
|
return RemoteTable(table, self.db_name)
|
||||||
|
|
||||||
|
def clone_table(
|
||||||
|
self,
|
||||||
|
target_table_name: str,
|
||||||
|
source_uri: str,
|
||||||
|
*,
|
||||||
|
target_namespace: List[str] = [],
|
||||||
|
source_version: Optional[int] = None,
|
||||||
|
source_tag: Optional[str] = None,
|
||||||
|
is_shallow: bool = True,
|
||||||
|
) -> Table:
|
||||||
|
"""Clone a table from a source table.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
target_table_name: str
|
||||||
|
The name of the target table to create.
|
||||||
|
source_uri: str
|
||||||
|
The URI of the source table to clone from.
|
||||||
|
target_namespace: List[str], optional
|
||||||
|
The namespace for the target table.
|
||||||
|
None or empty list represents root namespace.
|
||||||
|
source_version: int, optional
|
||||||
|
The version of the source table to clone.
|
||||||
|
source_tag: str, optional
|
||||||
|
The tag of the source table to clone.
|
||||||
|
is_shallow: bool, default True
|
||||||
|
Whether to perform a shallow clone (True) or deep clone (False).
|
||||||
|
Currently only shallow clone is supported.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
A RemoteTable object representing the cloned table.
|
||||||
|
"""
|
||||||
|
from .table import RemoteTable
|
||||||
|
|
||||||
|
table = LOOP.run(
|
||||||
|
self._conn.clone_table(
|
||||||
|
target_table_name,
|
||||||
|
source_uri,
|
||||||
|
target_namespace=target_namespace,
|
||||||
|
source_version=source_version,
|
||||||
|
source_tag=source_tag,
|
||||||
|
is_shallow=is_shallow,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return RemoteTable(table, self.db_name)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def create_table(
|
def create_table(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from .linear_combination import LinearCombinationReranker
|
|||||||
from .openai import OpenaiReranker
|
from .openai import OpenaiReranker
|
||||||
from .jinaai import JinaReranker
|
from .jinaai import JinaReranker
|
||||||
from .rrf import RRFReranker
|
from .rrf import RRFReranker
|
||||||
|
from .mrr import MRRReranker
|
||||||
from .answerdotai import AnswerdotaiRerankers
|
from .answerdotai import AnswerdotaiRerankers
|
||||||
from .voyageai import VoyageAIReranker
|
from .voyageai import VoyageAIReranker
|
||||||
|
|
||||||
@@ -23,4 +24,5 @@ __all__ = [
|
|||||||
"RRFReranker",
|
"RRFReranker",
|
||||||
"AnswerdotaiRerankers",
|
"AnswerdotaiRerankers",
|
||||||
"VoyageAIReranker",
|
"VoyageAIReranker",
|
||||||
|
"MRRReranker",
|
||||||
]
|
]
|
||||||
|
|||||||
169
python/python/lancedb/rerankers/mrr.py
Normal file
169
python/python/lancedb/rerankers/mrr.py
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
|
|
||||||
|
from typing import Union, List, TYPE_CHECKING
|
||||||
|
import pyarrow as pa
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
from .base import Reranker
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from ..table import LanceVectorQueryBuilder
|
||||||
|
|
||||||
|
|
||||||
|
class MRRReranker(Reranker):
|
||||||
|
"""
|
||||||
|
Reranks the results using Mean Reciprocal Rank (MRR) algorithm based
|
||||||
|
on the scores of vector and FTS search.
|
||||||
|
Algorithm reference - https://en.wikipedia.org/wiki/Mean_reciprocal_rank
|
||||||
|
|
||||||
|
MRR calculates the average of reciprocal ranks across different search results.
|
||||||
|
For each document, it computes the reciprocal of its rank in each system,
|
||||||
|
then takes the mean of these reciprocal ranks as the final score.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
weight_vector : float, default 0.5
|
||||||
|
Weight for vector search results (0.0 to 1.0)
|
||||||
|
weight_fts : float, default 0.5
|
||||||
|
Weight for FTS search results (0.0 to 1.0)
|
||||||
|
Note: weight_vector + weight_fts should equal 1.0
|
||||||
|
return_score : str, default "relevance"
|
||||||
|
Options are "relevance" or "all"
|
||||||
|
The type of score to return. If "relevance", will return only the relevance
|
||||||
|
score. If "all", will return all scores from the vector and FTS search along
|
||||||
|
with the relevance score.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
weight_vector: float = 0.5,
|
||||||
|
weight_fts: float = 0.5,
|
||||||
|
return_score="relevance",
|
||||||
|
):
|
||||||
|
if not (0.0 <= weight_vector <= 1.0):
|
||||||
|
raise ValueError("weight_vector must be between 0.0 and 1.0")
|
||||||
|
if not (0.0 <= weight_fts <= 1.0):
|
||||||
|
raise ValueError("weight_fts must be between 0.0 and 1.0")
|
||||||
|
if abs(weight_vector + weight_fts - 1.0) > 1e-6:
|
||||||
|
raise ValueError("weight_vector + weight_fts must equal 1.0")
|
||||||
|
|
||||||
|
super().__init__(return_score)
|
||||||
|
self.weight_vector = weight_vector
|
||||||
|
self.weight_fts = weight_fts
|
||||||
|
|
||||||
|
def rerank_hybrid(
|
||||||
|
self,
|
||||||
|
query: str, # noqa: F821
|
||||||
|
vector_results: pa.Table,
|
||||||
|
fts_results: pa.Table,
|
||||||
|
):
|
||||||
|
vector_ids = vector_results["_rowid"].to_pylist() if vector_results else []
|
||||||
|
fts_ids = fts_results["_rowid"].to_pylist() if fts_results else []
|
||||||
|
|
||||||
|
# Maps result_id to list of (type, reciprocal_rank)
|
||||||
|
mrr_score_map = defaultdict(list)
|
||||||
|
|
||||||
|
if vector_ids:
|
||||||
|
for rank, result_id in enumerate(vector_ids, 1):
|
||||||
|
reciprocal_rank = 1.0 / rank
|
||||||
|
mrr_score_map[result_id].append(("vector", reciprocal_rank))
|
||||||
|
|
||||||
|
if fts_ids:
|
||||||
|
for rank, result_id in enumerate(fts_ids, 1):
|
||||||
|
reciprocal_rank = 1.0 / rank
|
||||||
|
mrr_score_map[result_id].append(("fts", reciprocal_rank))
|
||||||
|
|
||||||
|
final_mrr_scores = {}
|
||||||
|
for result_id, scores in mrr_score_map.items():
|
||||||
|
vector_rr = 0.0
|
||||||
|
fts_rr = 0.0
|
||||||
|
|
||||||
|
for score_type, reciprocal_rank in scores:
|
||||||
|
if score_type == "vector":
|
||||||
|
vector_rr = reciprocal_rank
|
||||||
|
elif score_type == "fts":
|
||||||
|
fts_rr = reciprocal_rank
|
||||||
|
|
||||||
|
# If a document doesn't appear, its reciprocal rank is 0
|
||||||
|
weighted_mrr = self.weight_vector * vector_rr + self.weight_fts * fts_rr
|
||||||
|
final_mrr_scores[result_id] = weighted_mrr
|
||||||
|
|
||||||
|
combined_results = self.merge_results(vector_results, fts_results)
|
||||||
|
combined_row_ids = combined_results["_rowid"].to_pylist()
|
||||||
|
relevance_scores = [final_mrr_scores[row_id] for row_id in combined_row_ids]
|
||||||
|
combined_results = combined_results.append_column(
|
||||||
|
"_relevance_score", pa.array(relevance_scores, type=pa.float32())
|
||||||
|
)
|
||||||
|
combined_results = combined_results.sort_by(
|
||||||
|
[("_relevance_score", "descending")]
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.score == "relevance":
|
||||||
|
combined_results = self._keep_relevance_score(combined_results)
|
||||||
|
|
||||||
|
return combined_results
|
||||||
|
|
||||||
|
def rerank_multivector(
|
||||||
|
self,
|
||||||
|
vector_results: Union[List[pa.Table], List["LanceVectorQueryBuilder"]],
|
||||||
|
query: str = None,
|
||||||
|
deduplicate: bool = True, # noqa: F821
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Reranks the results from multiple vector searches using MRR algorithm.
|
||||||
|
Each vector search result is treated as a separate ranking system,
|
||||||
|
and MRR calculates the mean of reciprocal ranks across all systems.
|
||||||
|
This cannot reuse rerank_hybrid because MRR semantics require treating
|
||||||
|
each vector result as a separate ranking system.
|
||||||
|
"""
|
||||||
|
if not all(isinstance(v, type(vector_results[0])) for v in vector_results):
|
||||||
|
raise ValueError(
|
||||||
|
"All elements in vector_results should be of the same type"
|
||||||
|
)
|
||||||
|
|
||||||
|
# avoid circular import
|
||||||
|
if type(vector_results[0]).__name__ == "LanceVectorQueryBuilder":
|
||||||
|
vector_results = [result.to_arrow() for result in vector_results]
|
||||||
|
elif not isinstance(vector_results[0], pa.Table):
|
||||||
|
raise ValueError(
|
||||||
|
"vector_results should be a list of pa.Table or LanceVectorQueryBuilder"
|
||||||
|
)
|
||||||
|
|
||||||
|
if not all("_rowid" in result.column_names for result in vector_results):
|
||||||
|
raise ValueError(
|
||||||
|
"'_rowid' is required for deduplication. \
|
||||||
|
add _rowid to search results like this: \
|
||||||
|
`search().with_row_id(True)`"
|
||||||
|
)
|
||||||
|
|
||||||
|
mrr_score_map = defaultdict(list)
|
||||||
|
|
||||||
|
for result_table in vector_results:
|
||||||
|
result_ids = result_table["_rowid"].to_pylist()
|
||||||
|
for rank, result_id in enumerate(result_ids, 1):
|
||||||
|
reciprocal_rank = 1.0 / rank
|
||||||
|
mrr_score_map[result_id].append(reciprocal_rank)
|
||||||
|
|
||||||
|
final_mrr_scores = {}
|
||||||
|
for result_id, reciprocal_ranks in mrr_score_map.items():
|
||||||
|
mean_rr = np.mean(reciprocal_ranks)
|
||||||
|
final_mrr_scores[result_id] = mean_rr
|
||||||
|
|
||||||
|
combined = pa.concat_tables(vector_results, **self._concat_tables_args)
|
||||||
|
combined = self._deduplicate(combined)
|
||||||
|
|
||||||
|
combined_row_ids = combined["_rowid"].to_pylist()
|
||||||
|
|
||||||
|
relevance_scores = [final_mrr_scores[row_id] for row_id in combined_row_ids]
|
||||||
|
combined = combined.append_column(
|
||||||
|
"_relevance_score", pa.array(relevance_scores, type=pa.float32())
|
||||||
|
)
|
||||||
|
combined = combined.sort_by([("_relevance_score", "descending")])
|
||||||
|
|
||||||
|
if self.score == "relevance":
|
||||||
|
combined = self._keep_relevance_score(combined)
|
||||||
|
|
||||||
|
return combined
|
||||||
@@ -1470,10 +1470,7 @@ class Table(ABC):
|
|||||||
be deleted unless they are at least 7 days old. If delete_unverified is True
|
be deleted unless they are at least 7 days old. If delete_unverified is True
|
||||||
then these files will be deleted regardless of their age.
|
then these files will be deleted regardless of their age.
|
||||||
retrain: bool, default False
|
retrain: bool, default False
|
||||||
If True, retrain the vector indices, this would refine the IVF clustering
|
This parameter is no longer used and is deprecated.
|
||||||
and quantization, which may improve the search accuracy. It's faster than
|
|
||||||
re-creating the index from scratch, so it's recommended to try this first,
|
|
||||||
when the data distribution has changed significantly.
|
|
||||||
|
|
||||||
Experimental API
|
Experimental API
|
||||||
----------------
|
----------------
|
||||||
@@ -2835,10 +2832,7 @@ class LanceTable(Table):
|
|||||||
be deleted unless they are at least 7 days old. If delete_unverified is True
|
be deleted unless they are at least 7 days old. If delete_unverified is True
|
||||||
then these files will be deleted regardless of their age.
|
then these files will be deleted regardless of their age.
|
||||||
retrain: bool, default False
|
retrain: bool, default False
|
||||||
If True, retrain the vector indices, this would refine the IVF clustering
|
This parameter is no longer used and is deprecated.
|
||||||
and quantization, which may improve the search accuracy. It's faster than
|
|
||||||
re-creating the index from scratch, so it's recommended to try this first,
|
|
||||||
when the data distribution has changed significantly.
|
|
||||||
|
|
||||||
Experimental API
|
Experimental API
|
||||||
----------------
|
----------------
|
||||||
@@ -4298,10 +4292,7 @@ class AsyncTable:
|
|||||||
be deleted unless they are at least 7 days old. If delete_unverified is True
|
be deleted unless they are at least 7 days old. If delete_unverified is True
|
||||||
then these files will be deleted regardless of their age.
|
then these files will be deleted regardless of their age.
|
||||||
retrain: bool, default False
|
retrain: bool, default False
|
||||||
If True, retrain the vector indices, this would refine the IVF clustering
|
This parameter is no longer used and is deprecated.
|
||||||
and quantization, which may improve the search accuracy. It's faster than
|
|
||||||
re-creating the index from scratch, so it's recommended to try this first,
|
|
||||||
when the data distribution has changed significantly.
|
|
||||||
|
|
||||||
Experimental API
|
Experimental API
|
||||||
----------------
|
----------------
|
||||||
@@ -4324,10 +4315,19 @@ class AsyncTable:
|
|||||||
cleanup_since_ms: Optional[int] = None
|
cleanup_since_ms: Optional[int] = None
|
||||||
if cleanup_older_than is not None:
|
if cleanup_older_than is not None:
|
||||||
cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
|
cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
|
||||||
|
|
||||||
|
if retrain:
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
warnings.warn(
|
||||||
|
"The 'retrain' parameter is deprecated and will be removed in a "
|
||||||
|
"future version.",
|
||||||
|
DeprecationWarning,
|
||||||
|
)
|
||||||
|
|
||||||
return await self._inner.optimize(
|
return await self._inner.optimize(
|
||||||
cleanup_since_ms=cleanup_since_ms,
|
cleanup_since_ms=cleanup_since_ms,
|
||||||
delete_unverified=delete_unverified,
|
delete_unverified=delete_unverified,
|
||||||
retrain=retrain,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
async def list_indices(self) -> Iterable[IndexConfig]:
|
async def list_indices(self) -> Iterable[IndexConfig]:
|
||||||
|
|||||||
@@ -747,15 +747,16 @@ def test_local_namespace_operations(tmp_path):
|
|||||||
# Create a local database connection
|
# Create a local database connection
|
||||||
db = lancedb.connect(tmp_path)
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
# Test list_namespaces returns empty list
|
# Test list_namespaces returns empty list for root namespace
|
||||||
namespaces = list(db.list_namespaces())
|
namespaces = list(db.list_namespaces())
|
||||||
assert namespaces == []
|
assert namespaces == []
|
||||||
|
|
||||||
# Test list_namespaces with parameters still returns empty list
|
# Test list_namespaces with non-empty namespace raises NotImplementedError
|
||||||
namespaces_with_params = list(
|
with pytest.raises(
|
||||||
db.list_namespaces(namespace=["test"], page_token="token", limit=5)
|
NotImplementedError,
|
||||||
)
|
match="Namespace operations are not supported for listing database",
|
||||||
assert namespaces_with_params == []
|
):
|
||||||
|
list(db.list_namespaces(namespace=["test"]))
|
||||||
|
|
||||||
|
|
||||||
def test_local_create_namespace_not_supported(tmp_path):
|
def test_local_create_namespace_not_supported(tmp_path):
|
||||||
@@ -830,3 +831,119 @@ def test_local_table_operations_with_namespace_raise_error(tmp_path):
|
|||||||
# Test table_names without namespace - should work normally
|
# Test table_names without namespace - should work normally
|
||||||
tables_root = list(db.table_names())
|
tables_root = list(db.table_names())
|
||||||
assert "test_table" in tables_root
|
assert "test_table" in tables_root
|
||||||
|
|
||||||
|
|
||||||
|
def test_clone_table_latest_version(tmp_path):
|
||||||
|
"""Test cloning a table with the latest version (default behavior)"""
|
||||||
|
import os
|
||||||
|
|
||||||
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
|
# Create source table with some data
|
||||||
|
data = [
|
||||||
|
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
|
||||||
|
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
|
||||||
|
]
|
||||||
|
source_table = db.create_table("source", data=data)
|
||||||
|
|
||||||
|
# Add more data to create a new version
|
||||||
|
more_data = [{"id": 3, "text": "test", "vector": [5.0, 6.0]}]
|
||||||
|
source_table.add(more_data)
|
||||||
|
|
||||||
|
# Clone the table (should get latest version with 3 rows)
|
||||||
|
source_uri = os.path.join(tmp_path, "source.lance")
|
||||||
|
cloned_table = db.clone_table("cloned", source_uri)
|
||||||
|
|
||||||
|
# Verify cloned table has all 3 rows
|
||||||
|
assert cloned_table.count_rows() == 3
|
||||||
|
assert "cloned" in db.table_names()
|
||||||
|
|
||||||
|
# Verify data matches
|
||||||
|
cloned_data = cloned_table.to_pandas()
|
||||||
|
assert len(cloned_data) == 3
|
||||||
|
assert set(cloned_data["id"].tolist()) == {1, 2, 3}
|
||||||
|
|
||||||
|
|
||||||
|
def test_clone_table_specific_version(tmp_path):
|
||||||
|
"""Test cloning a table from a specific version"""
|
||||||
|
import os
|
||||||
|
|
||||||
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
|
# Create source table with initial data
|
||||||
|
data = [
|
||||||
|
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
|
||||||
|
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
|
||||||
|
]
|
||||||
|
source_table = db.create_table("source", data=data)
|
||||||
|
|
||||||
|
# Get the initial version
|
||||||
|
initial_version = source_table.version
|
||||||
|
|
||||||
|
# Add more data to create a new version
|
||||||
|
more_data = [{"id": 3, "text": "test", "vector": [5.0, 6.0]}]
|
||||||
|
source_table.add(more_data)
|
||||||
|
|
||||||
|
# Verify source now has 3 rows
|
||||||
|
assert source_table.count_rows() == 3
|
||||||
|
|
||||||
|
# Clone from the initial version (should have only 2 rows)
|
||||||
|
source_uri = os.path.join(tmp_path, "source.lance")
|
||||||
|
cloned_table = db.clone_table("cloned", source_uri, source_version=initial_version)
|
||||||
|
|
||||||
|
# Verify cloned table has only the initial 2 rows
|
||||||
|
assert cloned_table.count_rows() == 2
|
||||||
|
cloned_data = cloned_table.to_pandas()
|
||||||
|
assert set(cloned_data["id"].tolist()) == {1, 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_clone_table_with_tag(tmp_path):
|
||||||
|
"""Test cloning a table from a tagged version"""
|
||||||
|
import os
|
||||||
|
|
||||||
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
|
# Create source table with initial data
|
||||||
|
data = [
|
||||||
|
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
|
||||||
|
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
|
||||||
|
]
|
||||||
|
source_table = db.create_table("source", data=data)
|
||||||
|
|
||||||
|
# Create a tag for the current version
|
||||||
|
source_table.tags.create("v1.0", source_table.version)
|
||||||
|
|
||||||
|
# Add more data after the tag
|
||||||
|
more_data = [{"id": 3, "text": "test", "vector": [5.0, 6.0]}]
|
||||||
|
source_table.add(more_data)
|
||||||
|
|
||||||
|
# Verify source now has 3 rows
|
||||||
|
assert source_table.count_rows() == 3
|
||||||
|
|
||||||
|
# Clone from the tagged version (should have only 2 rows)
|
||||||
|
source_uri = os.path.join(tmp_path, "source.lance")
|
||||||
|
cloned_table = db.clone_table("cloned", source_uri, source_tag="v1.0")
|
||||||
|
|
||||||
|
# Verify cloned table has only the tagged version's 2 rows
|
||||||
|
assert cloned_table.count_rows() == 2
|
||||||
|
cloned_data = cloned_table.to_pandas()
|
||||||
|
assert set(cloned_data["id"].tolist()) == {1, 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_clone_table_deep_clone_fails(tmp_path):
|
||||||
|
"""Test that deep clone raises an unsupported error"""
|
||||||
|
import os
|
||||||
|
|
||||||
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
|
# Create source table with some data
|
||||||
|
data = [
|
||||||
|
{"id": 1, "text": "hello", "vector": [1.0, 2.0]},
|
||||||
|
{"id": 2, "text": "world", "vector": [3.0, 4.0]},
|
||||||
|
]
|
||||||
|
db.create_table("source", data=data)
|
||||||
|
|
||||||
|
# Try to create a deep clone (should fail)
|
||||||
|
source_uri = os.path.join(tmp_path, "source.lance")
|
||||||
|
with pytest.raises(Exception, match="Deep clone is not yet implemented"):
|
||||||
|
db.clone_table("cloned", source_uri, is_shallow=False)
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from lancedb.rerankers import (
|
|||||||
JinaReranker,
|
JinaReranker,
|
||||||
AnswerdotaiRerankers,
|
AnswerdotaiRerankers,
|
||||||
VoyageAIReranker,
|
VoyageAIReranker,
|
||||||
|
MRRReranker,
|
||||||
)
|
)
|
||||||
from lancedb.table import LanceTable
|
from lancedb.table import LanceTable
|
||||||
|
|
||||||
@@ -46,6 +47,7 @@ def get_test_table(tmp_path, use_tantivy):
|
|||||||
db,
|
db,
|
||||||
"my_table",
|
"my_table",
|
||||||
schema=MyTable,
|
schema=MyTable,
|
||||||
|
mode="overwrite",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Need to test with a bunch of phrases to make sure sorting is consistent
|
# Need to test with a bunch of phrases to make sure sorting is consistent
|
||||||
@@ -96,7 +98,7 @@ def get_test_table(tmp_path, use_tantivy):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Create a fts index
|
# Create a fts index
|
||||||
table.create_fts_index("text", use_tantivy=use_tantivy)
|
table.create_fts_index("text", use_tantivy=use_tantivy, replace=True)
|
||||||
|
|
||||||
return table, MyTable
|
return table, MyTable
|
||||||
|
|
||||||
@@ -320,6 +322,34 @@ def test_rrf_reranker(tmp_path, use_tantivy):
|
|||||||
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
|
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("use_tantivy", [True, False])
|
||||||
|
def test_mrr_reranker(tmp_path, use_tantivy):
|
||||||
|
reranker = MRRReranker()
|
||||||
|
_run_test_hybrid_reranker(reranker, tmp_path, use_tantivy)
|
||||||
|
|
||||||
|
# Test multi-vector part
|
||||||
|
table, schema = get_test_table(tmp_path, use_tantivy)
|
||||||
|
query = "single player experience"
|
||||||
|
rs1 = table.search(query, vector_column_name="vector").limit(10).with_row_id(True)
|
||||||
|
rs2 = (
|
||||||
|
table.search(query, vector_column_name="meta_vector")
|
||||||
|
.limit(10)
|
||||||
|
.with_row_id(True)
|
||||||
|
)
|
||||||
|
result = reranker.rerank_multivector([rs1, rs2])
|
||||||
|
assert "_relevance_score" in result.column_names
|
||||||
|
assert len(result) <= 20
|
||||||
|
|
||||||
|
if len(result) > 1:
|
||||||
|
assert np.all(np.diff(result.column("_relevance_score").to_numpy()) <= 0), (
|
||||||
|
"The _relevance_score should be descending."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Test with duplicate results
|
||||||
|
result_deduped = reranker.rerank_multivector([rs1, rs2, rs1])
|
||||||
|
assert len(result_deduped) == len(result)
|
||||||
|
|
||||||
|
|
||||||
def test_rrf_reranker_distance():
|
def test_rrf_reranker_distance():
|
||||||
data = pa.table(
|
data = pa.table(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -163,6 +163,34 @@ impl Connection {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[pyo3(signature = (target_table_name, source_uri, target_namespace=vec![], source_version=None, source_tag=None, is_shallow=true))]
|
||||||
|
pub fn clone_table(
|
||||||
|
self_: PyRef<'_, Self>,
|
||||||
|
target_table_name: String,
|
||||||
|
source_uri: String,
|
||||||
|
target_namespace: Vec<String>,
|
||||||
|
source_version: Option<u64>,
|
||||||
|
source_tag: Option<String>,
|
||||||
|
is_shallow: bool,
|
||||||
|
) -> PyResult<Bound<'_, PyAny>> {
|
||||||
|
let inner = self_.get_inner()?.clone();
|
||||||
|
|
||||||
|
let mut builder = inner.clone_table(target_table_name, source_uri);
|
||||||
|
builder = builder.target_namespace(target_namespace);
|
||||||
|
if let Some(version) = source_version {
|
||||||
|
builder = builder.source_version(version);
|
||||||
|
}
|
||||||
|
if let Some(tag) = source_tag {
|
||||||
|
builder = builder.source_tag(tag);
|
||||||
|
}
|
||||||
|
builder = builder.is_shallow(is_shallow);
|
||||||
|
|
||||||
|
future_into_py(self_.py(), async move {
|
||||||
|
let table = builder.execute().await.infer_error()?;
|
||||||
|
Ok(Table::new(table))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[pyo3(signature = (cur_name, new_name, cur_namespace=vec![], new_namespace=vec![]))]
|
#[pyo3(signature = (cur_name, new_name, cur_namespace=vec![], new_namespace=vec![]))]
|
||||||
pub fn rename_table(
|
pub fn rename_table(
|
||||||
self_: PyRef<'_, Self>,
|
self_: PyRef<'_, Self>,
|
||||||
|
|||||||
@@ -591,12 +591,11 @@ impl Table {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Optimize the on-disk data by compacting and pruning old data, for better performance.
|
/// Optimize the on-disk data by compacting and pruning old data, for better performance.
|
||||||
#[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None, retrain=None))]
|
#[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None))]
|
||||||
pub fn optimize(
|
pub fn optimize(
|
||||||
self_: PyRef<'_, Self>,
|
self_: PyRef<'_, Self>,
|
||||||
cleanup_since_ms: Option<u64>,
|
cleanup_since_ms: Option<u64>,
|
||||||
delete_unverified: Option<bool>,
|
delete_unverified: Option<bool>,
|
||||||
retrain: Option<bool>,
|
|
||||||
) -> PyResult<Bound<'_, PyAny>> {
|
) -> PyResult<Bound<'_, PyAny>> {
|
||||||
let inner = self_.inner_ref()?.clone();
|
let inner = self_.inner_ref()?.clone();
|
||||||
let older_than = if let Some(ms) = cleanup_since_ms {
|
let older_than = if let Some(ms) = cleanup_since_ms {
|
||||||
@@ -632,10 +631,9 @@ impl Table {
|
|||||||
.prune
|
.prune
|
||||||
.unwrap();
|
.unwrap();
|
||||||
inner
|
inner
|
||||||
.optimize(lancedb::table::OptimizeAction::Index(match retrain {
|
.optimize(lancedb::table::OptimizeAction::Index(
|
||||||
Some(true) => OptimizeOptions::retrain(),
|
OptimizeOptions::default(),
|
||||||
_ => OptimizeOptions::default(),
|
))
|
||||||
}))
|
|
||||||
.await
|
.await
|
||||||
.infer_error()?;
|
.infer_error()?;
|
||||||
Ok(OptimizeStats {
|
Ok(OptimizeStats {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.22.1-beta.1"
|
version = "0.22.1-beta.3"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -17,9 +17,9 @@ use crate::database::listing::{
|
|||||||
ListingDatabase, OPT_NEW_TABLE_STORAGE_VERSION, OPT_NEW_TABLE_V2_MANIFEST_PATHS,
|
ListingDatabase, OPT_NEW_TABLE_STORAGE_VERSION, OPT_NEW_TABLE_V2_MANIFEST_PATHS,
|
||||||
};
|
};
|
||||||
use crate::database::{
|
use crate::database::{
|
||||||
CreateNamespaceRequest, CreateTableData, CreateTableMode, CreateTableRequest, Database,
|
CloneTableRequest, CreateNamespaceRequest, CreateTableData, CreateTableMode,
|
||||||
DatabaseOptions, DropNamespaceRequest, ListNamespacesRequest, OpenTableRequest,
|
CreateTableRequest, Database, DatabaseOptions, DropNamespaceRequest, ListNamespacesRequest,
|
||||||
TableNamesRequest,
|
OpenTableRequest, TableNamesRequest,
|
||||||
};
|
};
|
||||||
use crate::embeddings::{
|
use crate::embeddings::{
|
||||||
EmbeddingDefinition, EmbeddingFunction, EmbeddingRegistry, MemoryRegistry, WithEmbeddings,
|
EmbeddingDefinition, EmbeddingFunction, EmbeddingRegistry, MemoryRegistry, WithEmbeddings,
|
||||||
@@ -469,6 +469,62 @@ impl OpenTableBuilder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Builder for cloning a table.
|
||||||
|
///
|
||||||
|
/// A shallow clone creates a new table that shares the underlying data files
|
||||||
|
/// with the source table but has its own independent manifest. Both the source
|
||||||
|
/// and cloned tables can evolve independently while initially sharing the same
|
||||||
|
/// data, deletion, and index files.
|
||||||
|
///
|
||||||
|
/// Use this builder to configure the clone operation before executing it.
|
||||||
|
pub struct CloneTableBuilder {
|
||||||
|
parent: Arc<dyn Database>,
|
||||||
|
request: CloneTableRequest,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CloneTableBuilder {
|
||||||
|
fn new(parent: Arc<dyn Database>, target_table_name: String, source_uri: String) -> Self {
|
||||||
|
Self {
|
||||||
|
parent,
|
||||||
|
request: CloneTableRequest::new(target_table_name, source_uri),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the source version to clone from
|
||||||
|
pub fn source_version(mut self, version: u64) -> Self {
|
||||||
|
self.request.source_version = Some(version);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the source tag to clone from
|
||||||
|
pub fn source_tag(mut self, tag: impl Into<String>) -> Self {
|
||||||
|
self.request.source_tag = Some(tag.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set the target namespace for the cloned table
|
||||||
|
pub fn target_namespace(mut self, namespace: Vec<String>) -> Self {
|
||||||
|
self.request.target_namespace = namespace;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set whether to perform a shallow clone (default: true)
|
||||||
|
///
|
||||||
|
/// When true, the cloned table shares data files with the source table.
|
||||||
|
/// When false, performs a deep clone (not yet implemented).
|
||||||
|
pub fn is_shallow(mut self, is_shallow: bool) -> Self {
|
||||||
|
self.request.is_shallow = is_shallow;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Execute the clone operation
|
||||||
|
pub async fn execute(self) -> Result<Table> {
|
||||||
|
Ok(Table::new(
|
||||||
|
self.parent.clone().clone_table(self.request).await?,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// A connection to LanceDB
|
/// A connection to LanceDB
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct Connection {
|
pub struct Connection {
|
||||||
@@ -575,6 +631,30 @@ impl Connection {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Clone a table in the database
|
||||||
|
///
|
||||||
|
/// Creates a new table by cloning from an existing source table.
|
||||||
|
/// By default, this performs a shallow clone where the new table shares
|
||||||
|
/// the underlying data files with the source table.
|
||||||
|
///
|
||||||
|
/// # Parameters
|
||||||
|
/// - `target_table_name`: The name of the new table to create
|
||||||
|
/// - `source_uri`: The URI of the source table to clone from
|
||||||
|
///
|
||||||
|
/// # Returns
|
||||||
|
/// A [`CloneTableBuilder`] that can be used to configure the clone operation
|
||||||
|
pub fn clone_table(
|
||||||
|
&self,
|
||||||
|
target_table_name: impl Into<String>,
|
||||||
|
source_uri: impl Into<String>,
|
||||||
|
) -> CloneTableBuilder {
|
||||||
|
CloneTableBuilder::new(
|
||||||
|
self.internal.clone(),
|
||||||
|
target_table_name.into(),
|
||||||
|
source_uri.into(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Rename a table in the database.
|
/// Rename a table in the database.
|
||||||
///
|
///
|
||||||
/// This is only supported in LanceDB Cloud.
|
/// This is only supported in LanceDB Cloud.
|
||||||
@@ -1281,4 +1361,50 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(other_schema, overwritten.schema().await.unwrap());
|
assert_eq!(other_schema, overwritten.schema().await.unwrap());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table() {
|
||||||
|
let tmp_dir = tempdir().unwrap();
|
||||||
|
let uri = tmp_dir.path().to_str().unwrap();
|
||||||
|
let db = connect(uri).execute().await.unwrap();
|
||||||
|
|
||||||
|
// Create a source table with some data
|
||||||
|
let mut batch_gen = BatchGenerator::new()
|
||||||
|
.col(Box::new(IncrementingInt32::new().named("id")))
|
||||||
|
.col(Box::new(IncrementingInt32::new().named("value")));
|
||||||
|
let reader = batch_gen.batches(5, 100);
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table("source_table", reader)
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Get the source table URI
|
||||||
|
let source_table_path = tmp_dir.path().join("source_table.lance");
|
||||||
|
let source_uri = source_table_path.to_str().unwrap();
|
||||||
|
|
||||||
|
// Clone the table
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table("cloned_table", source_uri)
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify the cloned table exists
|
||||||
|
let table_names = db.table_names().execute().await.unwrap();
|
||||||
|
assert!(table_names.contains(&"source_table".to_string()));
|
||||||
|
assert!(table_names.contains(&"cloned_table".to_string()));
|
||||||
|
|
||||||
|
// Verify the cloned table has the same schema
|
||||||
|
assert_eq!(
|
||||||
|
source_table.schema().await.unwrap(),
|
||||||
|
cloned_table.schema().await.unwrap()
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify the cloned table has the same data
|
||||||
|
let source_count = source_table.count_rows(None).await.unwrap();
|
||||||
|
let cloned_count = cloned_table.count_rows(None).await.unwrap();
|
||||||
|
assert_eq!(source_count, cloned_count);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -176,6 +176,42 @@ impl CreateTableRequest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Request to clone a table from a source table.
|
||||||
|
///
|
||||||
|
/// A shallow clone creates a new table that shares the underlying data files
|
||||||
|
/// with the source table but has its own independent manifest. This allows
|
||||||
|
/// both the source and cloned tables to evolve independently while initially
|
||||||
|
/// sharing the same data, deletion, and index files.
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
pub struct CloneTableRequest {
|
||||||
|
/// The name of the target table to create
|
||||||
|
pub target_table_name: String,
|
||||||
|
/// The namespace for the target table. Empty list represents root namespace.
|
||||||
|
pub target_namespace: Vec<String>,
|
||||||
|
/// The URI of the source table to clone from.
|
||||||
|
pub source_uri: String,
|
||||||
|
/// Optional version of the source table to clone.
|
||||||
|
pub source_version: Option<u64>,
|
||||||
|
/// Optional tag of the source table to clone.
|
||||||
|
pub source_tag: Option<String>,
|
||||||
|
/// Whether to perform a shallow clone (true) or deep clone (false). Defaults to true.
|
||||||
|
/// Currently only shallow clone is supported.
|
||||||
|
pub is_shallow: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CloneTableRequest {
|
||||||
|
pub fn new(target_table_name: String, source_uri: String) -> Self {
|
||||||
|
Self {
|
||||||
|
target_table_name,
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The `Database` trait defines the interface for database implementations.
|
/// The `Database` trait defines the interface for database implementations.
|
||||||
///
|
///
|
||||||
/// A database is responsible for managing tables and their metadata.
|
/// A database is responsible for managing tables and their metadata.
|
||||||
@@ -193,6 +229,13 @@ pub trait Database:
|
|||||||
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>>;
|
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>>;
|
||||||
/// Create a table in the database
|
/// Create a table in the database
|
||||||
async fn create_table(&self, request: CreateTableRequest) -> Result<Arc<dyn BaseTable>>;
|
async fn create_table(&self, request: CreateTableRequest) -> Result<Arc<dyn BaseTable>>;
|
||||||
|
/// Clone a table in the database.
|
||||||
|
///
|
||||||
|
/// Creates a shallow clone of the source table, sharing underlying data files
|
||||||
|
/// but with an independent manifest. Both tables can evolve separately after cloning.
|
||||||
|
///
|
||||||
|
/// See [`CloneTableRequest`] for detailed documentation and examples.
|
||||||
|
async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>>;
|
||||||
/// Open a table in the database
|
/// Open a table in the database
|
||||||
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>>;
|
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>>;
|
||||||
/// Rename a table in the database
|
/// Rename a table in the database
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ use std::fs::create_dir_all;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::{collections::HashMap, sync::Arc};
|
use std::{collections::HashMap, sync::Arc};
|
||||||
|
|
||||||
use lance::dataset::{ReadParams, WriteMode};
|
use lance::dataset::refs::Ref;
|
||||||
|
use lance::dataset::{builder::DatasetBuilder, ReadParams, WriteMode};
|
||||||
use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
|
use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
|
||||||
use lance_datafusion::utils::StreamingWriteSource;
|
use lance_datafusion::utils::StreamingWriteSource;
|
||||||
use lance_encoding::version::LanceFileVersion;
|
use lance_encoding::version::LanceFileVersion;
|
||||||
@@ -22,8 +23,8 @@ use crate::table::NativeTable;
|
|||||||
use crate::utils::validate_table_name;
|
use crate::utils::validate_table_name;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
BaseTable, CreateNamespaceRequest, CreateTableMode, CreateTableRequest, Database,
|
BaseTable, CloneTableRequest, CreateNamespaceRequest, CreateTableMode, CreateTableRequest,
|
||||||
DatabaseOptions, DropNamespaceRequest, ListNamespacesRequest, OpenTableRequest,
|
Database, DatabaseOptions, DropNamespaceRequest, ListNamespacesRequest, OpenTableRequest,
|
||||||
TableNamesRequest,
|
TableNamesRequest,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -684,6 +685,65 @@ impl Database for ListingDatabase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
|
if !request.target_namespace.is_empty() {
|
||||||
|
return Err(Error::NotSupported {
|
||||||
|
message: "Namespace parameter is not supported for listing database. Only root namespace is supported.".into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: support deep clone
|
||||||
|
if !request.is_shallow {
|
||||||
|
return Err(Error::NotSupported {
|
||||||
|
message: "Deep clone is not yet implemented".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_table_name(&request.target_table_name)?;
|
||||||
|
|
||||||
|
let storage_params = ObjectStoreParams {
|
||||||
|
storage_options: Some(self.storage_options.clone()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let read_params = ReadParams {
|
||||||
|
store_options: Some(storage_params.clone()),
|
||||||
|
session: Some(self.session.clone()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri)
|
||||||
|
.with_read_params(read_params.clone())
|
||||||
|
.load()
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::Lance { source: e })?;
|
||||||
|
|
||||||
|
let version_ref = match (request.source_version, request.source_tag) {
|
||||||
|
(Some(v), None) => Ok(Ref::Version(v)),
|
||||||
|
(None, Some(tag)) => Ok(Ref::Tag(tag)),
|
||||||
|
(None, None) => Ok(Ref::Version(source_dataset.version().version)),
|
||||||
|
_ => Err(Error::InvalidInput {
|
||||||
|
message: "Cannot specify both source_version and source_tag".to_string(),
|
||||||
|
}),
|
||||||
|
}?;
|
||||||
|
|
||||||
|
let target_uri = self.table_uri(&request.target_table_name)?;
|
||||||
|
source_dataset
|
||||||
|
.shallow_clone(&target_uri, version_ref, storage_params)
|
||||||
|
.await
|
||||||
|
.map_err(|e| Error::Lance { source: e })?;
|
||||||
|
|
||||||
|
let cloned_table = NativeTable::open_with_params(
|
||||||
|
&target_uri,
|
||||||
|
&request.target_table_name,
|
||||||
|
self.store_wrapper.clone(),
|
||||||
|
None,
|
||||||
|
self.read_consistency_interval,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
Ok(Arc::new(cloned_table))
|
||||||
|
}
|
||||||
|
|
||||||
async fn open_table(&self, mut request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn open_table(&self, mut request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
if !request.namespace.is_empty() {
|
if !request.namespace.is_empty() {
|
||||||
return Err(Error::NotSupported {
|
return Err(Error::NotSupported {
|
||||||
@@ -785,3 +845,694 @@ impl Database for ListingDatabase {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::connection::ConnectRequest;
|
||||||
|
use crate::database::{CreateTableData, CreateTableMode, CreateTableRequest};
|
||||||
|
use crate::table::{Table, TableDefinition};
|
||||||
|
use arrow_array::{Int32Array, RecordBatch, StringArray};
|
||||||
|
use arrow_schema::{DataType, Field, Schema};
|
||||||
|
use tempfile::tempdir;
|
||||||
|
|
||||||
|
async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
|
||||||
|
let tempdir = tempdir().unwrap();
|
||||||
|
let uri = tempdir.path().to_str().unwrap();
|
||||||
|
|
||||||
|
let request = ConnectRequest {
|
||||||
|
uri: uri.to_string(),
|
||||||
|
#[cfg(feature = "remote")]
|
||||||
|
client_config: Default::default(),
|
||||||
|
options: Default::default(),
|
||||||
|
read_consistency_interval: None,
|
||||||
|
session: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let db = ListingDatabase::connect_with_options(&request)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
(tempdir, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_basic() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table with schema
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("name", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table(CreateTableRequest {
|
||||||
|
name: "source_table".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema.clone())),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Get the source table URI
|
||||||
|
let source_uri = db.table_uri("source_table").unwrap();
|
||||||
|
|
||||||
|
// Clone the table
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned_table".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri: source_uri.clone(),
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify both tables exist
|
||||||
|
let table_names = db.table_names(TableNamesRequest::default()).await.unwrap();
|
||||||
|
assert!(table_names.contains(&"source_table".to_string()));
|
||||||
|
assert!(table_names.contains(&"cloned_table".to_string()));
|
||||||
|
|
||||||
|
// Verify schemas match
|
||||||
|
assert_eq!(
|
||||||
|
source_table.schema().await.unwrap(),
|
||||||
|
cloned_table.schema().await.unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_data() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table with actual data
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("name", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
let batch = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![1, 2, 3])),
|
||||||
|
Arc::new(StringArray::from(vec!["a", "b", "c"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch)],
|
||||||
|
schema.clone(),
|
||||||
|
));
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table(CreateTableRequest {
|
||||||
|
name: "source_with_data".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Data(reader),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("source_with_data").unwrap();
|
||||||
|
|
||||||
|
// Clone the table
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned_with_data".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify data counts match
|
||||||
|
let source_count = source_table.count_rows(None).await.unwrap();
|
||||||
|
let cloned_count = cloned_table.count_rows(None).await.unwrap();
|
||||||
|
assert_eq!(source_count, cloned_count);
|
||||||
|
assert_eq!(source_count, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_storage_options() {
|
||||||
|
let tempdir = tempdir().unwrap();
|
||||||
|
let uri = tempdir.path().to_str().unwrap();
|
||||||
|
|
||||||
|
// Create database with storage options
|
||||||
|
let mut options = HashMap::new();
|
||||||
|
options.insert("test_option".to_string(), "test_value".to_string());
|
||||||
|
|
||||||
|
let request = ConnectRequest {
|
||||||
|
uri: uri.to_string(),
|
||||||
|
#[cfg(feature = "remote")]
|
||||||
|
client_config: Default::default(),
|
||||||
|
options: options.clone(),
|
||||||
|
read_consistency_interval: None,
|
||||||
|
session: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let db = ListingDatabase::connect_with_options(&request)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Create source table
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("source").unwrap();
|
||||||
|
|
||||||
|
// Clone should work with storage options
|
||||||
|
let cloned = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(cloned.is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_deep_not_supported() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("source").unwrap();
|
||||||
|
|
||||||
|
// Try deep clone (should fail)
|
||||||
|
let result = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: false, // Request deep clone
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert!(matches!(
|
||||||
|
result.unwrap_err(),
|
||||||
|
Error::NotSupported { message } if message.contains("Deep clone")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_namespace_not_supported() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("source").unwrap();
|
||||||
|
|
||||||
|
// Try clone with namespace (should fail for listing database)
|
||||||
|
let result = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned".to_string(),
|
||||||
|
target_namespace: vec!["namespace".to_string()], // Non-empty namespace
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert!(matches!(
|
||||||
|
result.unwrap_err(),
|
||||||
|
Error::NotSupported { message } if message.contains("Namespace parameter is not supported")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_invalid_target_name() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("source").unwrap();
|
||||||
|
|
||||||
|
// Try clone with invalid target name
|
||||||
|
let result = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "invalid/name".to_string(), // Invalid name with slash
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_source_not_found() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Try to clone from non-existent source
|
||||||
|
let result = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri: "/nonexistent/table.lance".to_string(),
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_version_and_tag_error() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Empty(TableDefinition::new_from_schema(schema)),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("source").unwrap();
|
||||||
|
|
||||||
|
// Try clone with both version and tag (should fail)
|
||||||
|
let result = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: Some(1),
|
||||||
|
source_tag: Some("v1.0".to_string()),
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert!(matches!(
|
||||||
|
result.unwrap_err(),
|
||||||
|
Error::InvalidInput { message } if message.contains("Cannot specify both source_version and source_tag")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_specific_version() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table with initial data
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("value", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
let batch1 = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![1, 2])),
|
||||||
|
Arc::new(StringArray::from(vec!["a", "b"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch1)],
|
||||||
|
schema.clone(),
|
||||||
|
));
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table(CreateTableRequest {
|
||||||
|
name: "versioned_source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Data(reader),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Get the initial version
|
||||||
|
let initial_version = source_table.version().await.unwrap();
|
||||||
|
|
||||||
|
// Add more data to create a new version
|
||||||
|
let batch2 = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![3, 4])),
|
||||||
|
Arc::new(StringArray::from(vec!["c", "d"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_table_obj = Table::new(source_table.clone());
|
||||||
|
source_table_obj
|
||||||
|
.add(Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch2)],
|
||||||
|
schema.clone(),
|
||||||
|
)))
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify source table now has 4 rows
|
||||||
|
assert_eq!(source_table.count_rows(None).await.unwrap(), 4);
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("versioned_source").unwrap();
|
||||||
|
|
||||||
|
// Clone from the initial version (should have only 2 rows)
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned_from_version".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: Some(initial_version),
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify cloned table has only the initial 2 rows
|
||||||
|
assert_eq!(cloned_table.count_rows(None).await.unwrap(), 2);
|
||||||
|
|
||||||
|
// Source table should still have 4 rows
|
||||||
|
assert_eq!(source_table.count_rows(None).await.unwrap(), 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_tag() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table with initial data
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("value", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
let batch1 = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![1, 2])),
|
||||||
|
Arc::new(StringArray::from(vec!["a", "b"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch1)],
|
||||||
|
schema.clone(),
|
||||||
|
));
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table(CreateTableRequest {
|
||||||
|
name: "tagged_source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Data(reader),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Create a tag for the current version
|
||||||
|
let source_table_obj = Table::new(source_table.clone());
|
||||||
|
let mut tags = source_table_obj.tags().await.unwrap();
|
||||||
|
tags.create("v1.0", source_table.version().await.unwrap())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Add more data after the tag
|
||||||
|
let batch2 = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![3, 4])),
|
||||||
|
Arc::new(StringArray::from(vec!["c", "d"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_table_obj = Table::new(source_table.clone());
|
||||||
|
source_table_obj
|
||||||
|
.add(Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch2)],
|
||||||
|
schema.clone(),
|
||||||
|
)))
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Source table should have 4 rows
|
||||||
|
assert_eq!(source_table.count_rows(None).await.unwrap(), 4);
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("tagged_source").unwrap();
|
||||||
|
|
||||||
|
// Clone from the tag (should have only 2 rows)
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned_from_tag".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: Some("v1.0".to_string()),
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify cloned table has only the tagged version's 2 rows
|
||||||
|
assert_eq!(cloned_table.count_rows(None).await.unwrap(), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_cloned_tables_evolve_independently() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table with initial data
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("value", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
let batch1 = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![1, 2])),
|
||||||
|
Arc::new(StringArray::from(vec!["a", "b"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch1)],
|
||||||
|
schema.clone(),
|
||||||
|
));
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table(CreateTableRequest {
|
||||||
|
name: "independent_source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Data(reader),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("independent_source").unwrap();
|
||||||
|
|
||||||
|
// Clone the table
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "independent_clone".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Both should start with 2 rows
|
||||||
|
assert_eq!(source_table.count_rows(None).await.unwrap(), 2);
|
||||||
|
assert_eq!(cloned_table.count_rows(None).await.unwrap(), 2);
|
||||||
|
|
||||||
|
// Add data to the cloned table
|
||||||
|
let batch_clone = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![3, 4, 5])),
|
||||||
|
Arc::new(StringArray::from(vec!["c", "d", "e"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let cloned_table_obj = Table::new(cloned_table.clone());
|
||||||
|
cloned_table_obj
|
||||||
|
.add(Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch_clone)],
|
||||||
|
schema.clone(),
|
||||||
|
)))
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Add different data to the source table
|
||||||
|
let batch_source = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![
|
||||||
|
Arc::new(Int32Array::from(vec![10, 11])),
|
||||||
|
Arc::new(StringArray::from(vec!["x", "y"])),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_table_obj = Table::new(source_table.clone());
|
||||||
|
source_table_obj
|
||||||
|
.add(Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch_source)],
|
||||||
|
schema.clone(),
|
||||||
|
)))
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Verify they have evolved independently
|
||||||
|
assert_eq!(source_table.count_rows(None).await.unwrap(), 4); // 2 + 2
|
||||||
|
assert_eq!(cloned_table.count_rows(None).await.unwrap(), 5); // 2 + 3
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_latest_version() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
// Create a source table with initial data
|
||||||
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
|
|
||||||
|
let batch1 =
|
||||||
|
RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2]))])
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch1)],
|
||||||
|
schema.clone(),
|
||||||
|
));
|
||||||
|
|
||||||
|
let source_table = db
|
||||||
|
.create_table(CreateTableRequest {
|
||||||
|
name: "latest_version_source".to_string(),
|
||||||
|
namespace: vec![],
|
||||||
|
data: CreateTableData::Data(reader),
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Add more data to create new versions
|
||||||
|
for i in 0..3 {
|
||||||
|
let batch = RecordBatch::try_new(
|
||||||
|
schema.clone(),
|
||||||
|
vec![Arc::new(Int32Array::from(vec![i * 10, i * 10 + 1]))],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let source_table_obj = Table::new(source_table.clone());
|
||||||
|
source_table_obj
|
||||||
|
.add(Box::new(arrow_array::RecordBatchIterator::new(
|
||||||
|
vec![Ok(batch)],
|
||||||
|
schema.clone(),
|
||||||
|
)))
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Source should have 8 rows total (2 + 2 + 2 + 2)
|
||||||
|
let source_count = source_table.count_rows(None).await.unwrap();
|
||||||
|
assert_eq!(source_count, 8);
|
||||||
|
|
||||||
|
let source_uri = db.table_uri("latest_version_source").unwrap();
|
||||||
|
|
||||||
|
// Clone without specifying version or tag (should get latest)
|
||||||
|
let cloned_table = db
|
||||||
|
.clone_table(CloneTableRequest {
|
||||||
|
target_table_name: "cloned_latest".to_string(),
|
||||||
|
target_namespace: vec![],
|
||||||
|
source_uri,
|
||||||
|
source_version: None,
|
||||||
|
source_tag: None,
|
||||||
|
is_shallow: true,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Cloned table should have all 8 rows from the latest version
|
||||||
|
assert_eq!(cloned_table.count_rows(None).await.unwrap(), 8);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
//! values
|
//! values
|
||||||
use std::cmp::max;
|
use std::cmp::max;
|
||||||
|
|
||||||
use lance::table::format::{Index, Manifest};
|
use lance::table::format::{IndexMetadata, Manifest};
|
||||||
|
|
||||||
use crate::DistanceType;
|
use crate::DistanceType;
|
||||||
|
|
||||||
@@ -19,7 +19,7 @@ pub struct VectorIndex {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl VectorIndex {
|
impl VectorIndex {
|
||||||
pub fn new_from_format(manifest: &Manifest, index: &Index) -> Self {
|
pub fn new_from_format(manifest: &Manifest, index: &IndexMetadata) -> Self {
|
||||||
let fields = index
|
let fields = index
|
||||||
.fields
|
.fields
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -14,9 +14,9 @@ use serde::Deserialize;
|
|||||||
use tokio::task::spawn_blocking;
|
use tokio::task::spawn_blocking;
|
||||||
|
|
||||||
use crate::database::{
|
use crate::database::{
|
||||||
CreateNamespaceRequest, CreateTableData, CreateTableMode, CreateTableRequest, Database,
|
CloneTableRequest, CreateNamespaceRequest, CreateTableData, CreateTableMode,
|
||||||
DatabaseOptions, DropNamespaceRequest, ListNamespacesRequest, OpenTableRequest,
|
CreateTableRequest, Database, DatabaseOptions, DropNamespaceRequest, ListNamespacesRequest,
|
||||||
TableNamesRequest,
|
OpenTableRequest, TableNamesRequest,
|
||||||
};
|
};
|
||||||
use crate::error::Result;
|
use crate::error::Result;
|
||||||
use crate::table::BaseTable;
|
use crate::table::BaseTable;
|
||||||
@@ -27,6 +27,18 @@ use super::table::RemoteTable;
|
|||||||
use super::util::{batches_to_ipc_bytes, parse_server_version};
|
use super::util::{batches_to_ipc_bytes, parse_server_version};
|
||||||
use super::ARROW_STREAM_CONTENT_TYPE;
|
use super::ARROW_STREAM_CONTENT_TYPE;
|
||||||
|
|
||||||
|
// Request structure for the remote clone table API
|
||||||
|
#[derive(serde::Serialize)]
|
||||||
|
struct RemoteCloneTableRequest {
|
||||||
|
source_location: String,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
source_version: Option<u64>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
source_tag: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
is_shallow: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
// the versions of the server that we support
|
// the versions of the server that we support
|
||||||
// for any new feature that we need to change the SDK behavior, we should bump the server version,
|
// for any new feature that we need to change the SDK behavior, we should bump the server version,
|
||||||
// and add a feature flag as method of `ServerVersion` here.
|
// and add a feature flag as method of `ServerVersion` here.
|
||||||
@@ -430,6 +442,51 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
|||||||
Ok(table)
|
Ok(table)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
|
let table_identifier = build_table_identifier(
|
||||||
|
&request.target_table_name,
|
||||||
|
&request.target_namespace,
|
||||||
|
&self.client.id_delimiter,
|
||||||
|
);
|
||||||
|
|
||||||
|
let remote_request = RemoteCloneTableRequest {
|
||||||
|
source_location: request.source_uri,
|
||||||
|
source_version: request.source_version,
|
||||||
|
source_tag: request.source_tag,
|
||||||
|
is_shallow: Some(request.is_shallow),
|
||||||
|
};
|
||||||
|
|
||||||
|
let req = self
|
||||||
|
.client
|
||||||
|
.post(&format!("/v1/table/{}/clone", table_identifier.clone()))
|
||||||
|
.json(&remote_request);
|
||||||
|
|
||||||
|
let (request_id, rsp) = self.client.send(req).await?;
|
||||||
|
|
||||||
|
let status = rsp.status();
|
||||||
|
if status != StatusCode::OK {
|
||||||
|
let body = rsp.text().await.err_to_http(request_id.clone())?;
|
||||||
|
return Err(crate::Error::Http {
|
||||||
|
source: format!("Failed to clone table: {}", body).into(),
|
||||||
|
request_id,
|
||||||
|
status_code: Some(status),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let version = parse_server_version(&request_id, &rsp)?;
|
||||||
|
let cache_key = build_cache_key(&request.target_table_name, &request.target_namespace);
|
||||||
|
let table = Arc::new(RemoteTable::new(
|
||||||
|
self.client.clone(),
|
||||||
|
request.target_table_name.clone(),
|
||||||
|
request.target_namespace.clone(),
|
||||||
|
table_identifier,
|
||||||
|
version,
|
||||||
|
));
|
||||||
|
self.table_cache.insert(cache_key, table.clone()).await;
|
||||||
|
|
||||||
|
Ok(table)
|
||||||
|
}
|
||||||
|
|
||||||
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
let identifier =
|
let identifier =
|
||||||
build_table_identifier(&request.name, &request.namespace, &self.client.id_delimiter);
|
build_table_identifier(&request.name, &request.namespace, &self.client.id_delimiter);
|
||||||
@@ -1221,4 +1278,146 @@ mod tests {
|
|||||||
_ => panic!("Expected Runtime error from header provider"),
|
_ => panic!("Expected Runtime error from header provider"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table() {
|
||||||
|
let conn = Connection::new_with_handler(|request| {
|
||||||
|
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||||
|
assert_eq!(request.url().path(), "/v1/table/cloned_table/clone");
|
||||||
|
assert_eq!(
|
||||||
|
request.headers().get("Content-Type").unwrap(),
|
||||||
|
JSON_CONTENT_TYPE
|
||||||
|
);
|
||||||
|
|
||||||
|
let body = request.body().unwrap().as_bytes().unwrap();
|
||||||
|
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
|
assert_eq!(body["source_location"], "s3://bucket/source_table");
|
||||||
|
assert_eq!(body["is_shallow"], true);
|
||||||
|
|
||||||
|
http::Response::builder().status(200).body("").unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
let table = conn
|
||||||
|
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table.name(), "cloned_table");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_version() {
|
||||||
|
let conn = Connection::new_with_handler(|request| {
|
||||||
|
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||||
|
assert_eq!(request.url().path(), "/v1/table/cloned_table/clone");
|
||||||
|
|
||||||
|
let body = request.body().unwrap().as_bytes().unwrap();
|
||||||
|
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
|
assert_eq!(body["source_location"], "s3://bucket/source_table");
|
||||||
|
assert_eq!(body["source_version"], 42);
|
||||||
|
assert_eq!(body["is_shallow"], true);
|
||||||
|
|
||||||
|
http::Response::builder().status(200).body("").unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
let table = conn
|
||||||
|
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||||
|
.source_version(42)
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table.name(), "cloned_table");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_tag() {
|
||||||
|
let conn = Connection::new_with_handler(|request| {
|
||||||
|
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||||
|
assert_eq!(request.url().path(), "/v1/table/cloned_table/clone");
|
||||||
|
|
||||||
|
let body = request.body().unwrap().as_bytes().unwrap();
|
||||||
|
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
|
assert_eq!(body["source_location"], "s3://bucket/source_table");
|
||||||
|
assert_eq!(body["source_tag"], "v1.0");
|
||||||
|
assert_eq!(body["is_shallow"], true);
|
||||||
|
|
||||||
|
http::Response::builder().status(200).body("").unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
let table = conn
|
||||||
|
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||||
|
.source_tag("v1.0")
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table.name(), "cloned_table");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_deep_clone() {
|
||||||
|
let conn = Connection::new_with_handler(|request| {
|
||||||
|
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||||
|
assert_eq!(request.url().path(), "/v1/table/cloned_table/clone");
|
||||||
|
|
||||||
|
let body = request.body().unwrap().as_bytes().unwrap();
|
||||||
|
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
|
assert_eq!(body["source_location"], "s3://bucket/source_table");
|
||||||
|
assert_eq!(body["is_shallow"], false);
|
||||||
|
|
||||||
|
http::Response::builder().status(200).body("").unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
let table = conn
|
||||||
|
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||||
|
.is_shallow(false)
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table.name(), "cloned_table");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_with_namespace() {
|
||||||
|
let conn = Connection::new_with_handler(|request| {
|
||||||
|
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||||
|
assert_eq!(request.url().path(), "/v1/table/ns1$ns2$cloned_table/clone");
|
||||||
|
|
||||||
|
let body = request.body().unwrap().as_bytes().unwrap();
|
||||||
|
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||||
|
assert_eq!(body["source_location"], "s3://bucket/source_table");
|
||||||
|
assert_eq!(body["is_shallow"], true);
|
||||||
|
|
||||||
|
http::Response::builder().status(200).body("").unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
let table = conn
|
||||||
|
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||||
|
.target_namespace(vec!["ns1".to_string(), "ns2".to_string()])
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table.name(), "cloned_table");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_clone_table_error() {
|
||||||
|
let conn = Connection::new_with_handler(|_| {
|
||||||
|
http::Response::builder()
|
||||||
|
.status(500)
|
||||||
|
.body("Internal server error")
|
||||||
|
.unwrap()
|
||||||
|
});
|
||||||
|
|
||||||
|
let result = conn
|
||||||
|
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||||
|
.execute()
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(result.is_err());
|
||||||
|
if let Err(crate::Error::Http { source, .. }) = result {
|
||||||
|
assert!(source.to_string().contains("Failed to clone table"));
|
||||||
|
} else {
|
||||||
|
panic!("Expected HTTP error");
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1976,6 +1976,8 @@ impl NativeTable {
|
|||||||
/// Delete keys from the config
|
/// Delete keys from the config
|
||||||
pub async fn delete_config_keys(&self, delete_keys: &[&str]) -> Result<()> {
|
pub async fn delete_config_keys(&self, delete_keys: &[&str]) -> Result<()> {
|
||||||
let mut dataset = self.dataset.get_mut().await?;
|
let mut dataset = self.dataset.get_mut().await?;
|
||||||
|
// TODO: update this when we implement metadata APIs
|
||||||
|
#[allow(deprecated)]
|
||||||
dataset.delete_config_keys(delete_keys).await?;
|
dataset.delete_config_keys(delete_keys).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@@ -1986,6 +1988,8 @@ impl NativeTable {
|
|||||||
upsert_values: impl IntoIterator<Item = (String, String)>,
|
upsert_values: impl IntoIterator<Item = (String, String)>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let mut dataset = self.dataset.get_mut().await?;
|
let mut dataset = self.dataset.get_mut().await?;
|
||||||
|
// TODO: update this when we implement metadata APIs
|
||||||
|
#[allow(deprecated)]
|
||||||
dataset.replace_schema_metadata(upsert_values).await?;
|
dataset.replace_schema_metadata(upsert_values).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user