mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-27 23:12:58 +00:00
Compare commits
5 Commits
codex/upda
...
python-v0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
9d129c7e86 | ||
|
44878dd9a5 | ||
|
4b5bb2d76c | ||
|
|
434f4124fc | ||
|
03a1a99270 |
173
Cargo.lock
generated
173
Cargo.lock
generated
@@ -2,15 +2,6 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.25.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
@@ -1066,30 +1057,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "backon"
|
||||
version = "1.5.2"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d"
|
||||
checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"gloo-timers",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
"windows-link 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base16ct"
|
||||
version = "0.1.1"
|
||||
@@ -1987,9 +1963,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion"
|
||||
version = "50.3.0"
|
||||
version = "50.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a"
|
||||
checksum = "fc6759cf9ef57c5c469e4027ac4b4cfa746e06a0f5472c2b922b6a403c2a64c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -3126,8 +3102,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.2",
|
||||
@@ -3653,12 +3629,6 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.32.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.3"
|
||||
@@ -4251,7 +4221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
|
||||
dependencies = [
|
||||
"equivalent",
|
||||
"hashbrown 0.16.0",
|
||||
"hashbrown 0.15.5",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
@@ -4300,17 +4270,6 @@ version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
|
||||
|
||||
[[package]]
|
||||
name = "io-uring"
|
||||
version = "0.7.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b"
|
||||
dependencies = [
|
||||
"bitflags 2.9.4",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.11.0"
|
||||
@@ -4501,8 +4460,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4567,8 +4526,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4586,8 +4545,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"paste",
|
||||
@@ -4596,8 +4555,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4633,8 +4592,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4664,8 +4623,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4682,8 +4641,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4720,8 +4679,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4753,8 +4712,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-types",
|
||||
@@ -4765,8 +4724,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4827,8 +4786,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4868,8 +4827,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4885,8 +4844,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -4898,8 +4857,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -4937,8 +4896,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4977,8 +4936,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "1.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.13#e09f99948820a56851873672fe88417d88ea8fca"
|
||||
version = "1.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v1.0.0-beta.16#ff89675ebaca362412c3ec9d62d2de3885a57727"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5818,15 +5777,6 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.37.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object_store"
|
||||
version = "0.12.4"
|
||||
@@ -5867,12 +5817,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "object_store_opendal"
|
||||
version = "0.54.0"
|
||||
version = "0.55.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ce697ee723fdc3eaf6c457abf4059034be15167022b18b619993802cd1443d5"
|
||||
checksum = "113ab0769e972eee585e57407b98de08bda5354fa28e8ba4d89038d6cb6a8991"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"object_store",
|
||||
"opendal",
|
||||
@@ -5922,30 +5873,31 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "opendal"
|
||||
version = "0.54.0"
|
||||
version = "0.55.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a"
|
||||
checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"backon",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc32c",
|
||||
"futures",
|
||||
"getrandom 0.2.16",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"jiff",
|
||||
"log",
|
||||
"md-5",
|
||||
"percent-encoding",
|
||||
"quick-xml 0.37.5",
|
||||
"quick-xml 0.38.3",
|
||||
"reqsign",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"tokio",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -6760,8 +6712,8 @@ version = "0.13.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"heck 0.4.1",
|
||||
"itertools 0.12.1",
|
||||
"log",
|
||||
"multimap",
|
||||
"once_cell",
|
||||
@@ -6781,7 +6733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.12.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.106",
|
||||
@@ -7331,9 +7283,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.12.23"
|
||||
version = "0.12.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
|
||||
checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"base64 0.22.1",
|
||||
@@ -7515,12 +7467,6 @@ dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
@@ -8106,7 +8052,7 @@ version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.106",
|
||||
@@ -8813,29 +8759,26 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.47.1"
|
||||
version = "1.48.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
|
||||
checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"bytes",
|
||||
"io-uring",
|
||||
"libc",
|
||||
"mio",
|
||||
"parking_lot",
|
||||
"pin-project-lite",
|
||||
"signal-hook-registry",
|
||||
"slab",
|
||||
"socket2 0.6.0",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.5.0"
|
||||
version = "2.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
|
||||
checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
40
Cargo.toml
40
Cargo.toml
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.78.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=1.0.0-beta.13", default-features = false, "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=1.0.0-beta.13", default-features = false, "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=1.0.0-beta.13", "features" = ["dir-aws", "dir-gcp", "dir-azure", "dir-oss", "rest"], "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=1.0.0-beta.13", "tag" = "v1.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=1.0.0-beta.16", default-features = false, "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=1.0.0-beta.16", default-features = false, "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=1.0.0-beta.16", default-features = false, "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=1.0.0-beta.16", "tag" = "v1.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "56.2", optional = false }
|
||||
@@ -40,12 +40,12 @@ arrow-schema = "56.2"
|
||||
arrow-select = "56.2"
|
||||
arrow-cast = "56.2"
|
||||
async-trait = "0"
|
||||
datafusion = { version = "50.3", default-features = false }
|
||||
datafusion-catalog = "50.3"
|
||||
datafusion-common = { version = "50.3", default-features = false }
|
||||
datafusion-execution = "50.3"
|
||||
datafusion-expr = "50.3"
|
||||
datafusion-physical-plan = "50.3"
|
||||
datafusion = { version = "50.1", default-features = false }
|
||||
datafusion-catalog = "50.1"
|
||||
datafusion-common = { version = "50.1", default-features = false }
|
||||
datafusion-execution = "50.1"
|
||||
datafusion-expr = "50.1"
|
||||
datafusion-physical-plan = "50.1"
|
||||
env_logger = "0.11"
|
||||
half = { "version" = "2.6.0", default-features = false, features = [
|
||||
"num-traits",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.25.4-beta.2"
|
||||
current_version = "0.25.4-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.25.4-beta.2"
|
||||
version = "0.25.4-beta.3"
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
license.workspace = true
|
||||
|
||||
@@ -10,7 +10,7 @@ dependencies = [
|
||||
"pyarrow>=16",
|
||||
"pydantic>=1.10",
|
||||
"tqdm>=4.27.0",
|
||||
"lance-namespace>=0.0.21"
|
||||
"lance-namespace>=0.2.1"
|
||||
]
|
||||
description = "lancedb"
|
||||
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
|
||||
@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"
|
||||
|
||||
[project.optional-dependencies]
|
||||
pylance = [
|
||||
"pylance>=0.25",
|
||||
"pylance>=1.0.0b14",
|
||||
]
|
||||
tests = [
|
||||
"aiohttp",
|
||||
@@ -59,7 +59,7 @@ tests = [
|
||||
"polars>=0.19, <=1.3.0",
|
||||
"tantivy",
|
||||
"pyarrow-stubs",
|
||||
"pylance>=1.0.0b4",
|
||||
"pylance>=1.0.0b14",
|
||||
"requests",
|
||||
"datafusion",
|
||||
]
|
||||
|
||||
@@ -194,6 +194,10 @@ class DBConnection(EnforceOverrides):
|
||||
connection will be inherited by the table, but can be overridden here.
|
||||
See available options at
|
||||
<https://lancedb.com/docs/storage/>
|
||||
|
||||
To enable stable row IDs (row IDs remain stable after compaction,
|
||||
update, delete, and merges), set `new_table_enable_stable_row_ids`
|
||||
to `"true"` in storage_options when connecting to the database.
|
||||
data_storage_version: optional, str, default "stable"
|
||||
Deprecated. Set `storage_options` when connecting to the database and set
|
||||
`new_table_data_storage_version` in the options.
|
||||
@@ -1079,6 +1083,10 @@ class AsyncConnection(object):
|
||||
See available options at
|
||||
<https://lancedb.com/docs/storage/>
|
||||
|
||||
To enable stable row IDs (row IDs remain stable after compaction,
|
||||
update, delete, and merges), set `new_table_enable_stable_row_ids`
|
||||
to `"true"` in storage_options when connecting to the database.
|
||||
|
||||
Returns
|
||||
-------
|
||||
AsyncTable
|
||||
|
||||
@@ -2429,9 +2429,8 @@ class AsyncQueryBase(object):
|
||||
>>> from lancedb import connect_async
|
||||
>>> async def doctest_example():
|
||||
... conn = await connect_async("./.lancedb")
|
||||
... table = await conn.create_table("my_table", [{"vector": [99, 99]}])
|
||||
... query = [100, 100]
|
||||
... plan = await table.query().nearest_to([1, 2]).explain_plan(True)
|
||||
... table = await conn.create_table("my_table", [{"vector": [99.0, 99.0]}])
|
||||
... plan = await table.query().nearest_to([1.0, 2.0]).explain_plan(True)
|
||||
... print(plan)
|
||||
>>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
|
||||
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
|
||||
@@ -2440,6 +2439,7 @@ class AsyncQueryBase(object):
|
||||
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
|
||||
KNNVectorDistance: metric=l2
|
||||
LanceRead: uri=..., projection=[vector], ...
|
||||
<BLANKLINE>
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -3141,10 +3141,9 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
|
||||
>>> from lancedb.index import FTS
|
||||
>>> async def doctest_example():
|
||||
... conn = await connect_async("./.lancedb")
|
||||
... table = await conn.create_table("my_table", [{"vector": [99, 99], "text": "hello world"}])
|
||||
... table = await conn.create_table("my_table", [{"vector": [99.0, 99.0], "text": "hello world"}])
|
||||
... await table.create_index("text", config=FTS(with_position=False))
|
||||
... query = [100, 100]
|
||||
... plan = await table.query().nearest_to([1, 2]).nearest_to_text("hello").explain_plan(True)
|
||||
... plan = await table.query().nearest_to([1.0, 2.0]).nearest_to_text("hello").explain_plan(True)
|
||||
... print(plan)
|
||||
>>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
|
||||
Vector Search Plan:
|
||||
@@ -3418,9 +3417,8 @@ class BaseQueryBuilder(object):
|
||||
>>> from lancedb import connect_async
|
||||
>>> async def doctest_example():
|
||||
... conn = await connect_async("./.lancedb")
|
||||
... table = await conn.create_table("my_table", [{"vector": [99, 99]}])
|
||||
... query = [100, 100]
|
||||
... plan = await table.query().nearest_to([1, 2]).explain_plan(True)
|
||||
... table = await conn.create_table("my_table", [{"vector": [99.0, 99.0]}])
|
||||
... plan = await table.query().nearest_to([1.0, 2.0]).explain_plan(True)
|
||||
... print(plan)
|
||||
>>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
|
||||
ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
|
||||
@@ -3429,6 +3427,7 @@ class BaseQueryBuilder(object):
|
||||
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
|
||||
KNNVectorDistance: metric=l2
|
||||
LanceRead: uri=..., projection=[vector], ...
|
||||
<BLANKLINE>
|
||||
|
||||
Parameters
|
||||
----------
|
||||
|
||||
@@ -441,6 +441,150 @@ async def test_create_table_v2_manifest_paths_async(tmp_path):
|
||||
assert re.match(r"\d{20}\.manifest", manifest)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_table_stable_row_ids_via_storage_options(tmp_path):
|
||||
"""Test stable_row_ids via storage_options at connect time."""
|
||||
import lance
|
||||
|
||||
# Connect with stable row IDs enabled as default for new tables
|
||||
db_with = await lancedb.connect_async(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
|
||||
)
|
||||
# Connect without stable row IDs (default)
|
||||
db_without = await lancedb.connect_async(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "false"}
|
||||
)
|
||||
|
||||
# Create table using connection with stable row IDs enabled
|
||||
await db_with.create_table(
|
||||
"with_stable_via_opts",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
)
|
||||
lance_ds_with = lance.dataset(tmp_path / "with_stable_via_opts.lance")
|
||||
fragments_with = lance_ds_with.get_fragments()
|
||||
assert len(fragments_with) > 0
|
||||
assert fragments_with[0].metadata.row_id_meta is not None
|
||||
|
||||
# Create table using connection without stable row IDs
|
||||
await db_without.create_table(
|
||||
"without_stable_via_opts",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
)
|
||||
lance_ds_without = lance.dataset(tmp_path / "without_stable_via_opts.lance")
|
||||
fragments_without = lance_ds_without.get_fragments()
|
||||
assert len(fragments_without) > 0
|
||||
assert fragments_without[0].metadata.row_id_meta is None
|
||||
|
||||
|
||||
def test_create_table_stable_row_ids_via_storage_options_sync(tmp_path):
|
||||
"""Test that enable_stable_row_ids can be set via storage_options (sync API)."""
|
||||
# Connect with stable row IDs enabled as default for new tables
|
||||
db_with = lancedb.connect(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
|
||||
)
|
||||
# Connect without stable row IDs (default)
|
||||
db_without = lancedb.connect(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "false"}
|
||||
)
|
||||
|
||||
# Create table using connection with stable row IDs enabled
|
||||
tbl_with = db_with.create_table(
|
||||
"with_stable_sync",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
)
|
||||
lance_ds_with = tbl_with.to_lance()
|
||||
fragments_with = lance_ds_with.get_fragments()
|
||||
assert len(fragments_with) > 0
|
||||
assert fragments_with[0].metadata.row_id_meta is not None
|
||||
|
||||
# Create table using connection without stable row IDs
|
||||
tbl_without = db_without.create_table(
|
||||
"without_stable_sync",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
)
|
||||
lance_ds_without = tbl_without.to_lance()
|
||||
fragments_without = lance_ds_without.get_fragments()
|
||||
assert len(fragments_without) > 0
|
||||
assert fragments_without[0].metadata.row_id_meta is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_table_stable_row_ids_table_level_override(tmp_path):
|
||||
"""Test that stable_row_ids can be enabled/disabled at create_table level."""
|
||||
import lance
|
||||
|
||||
# Connect without any stable row ID setting
|
||||
db_default = await lancedb.connect_async(tmp_path)
|
||||
|
||||
# Connect with stable row IDs enabled at connection level
|
||||
db_with_stable = await lancedb.connect_async(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
|
||||
)
|
||||
|
||||
# Case 1: No connection setting, enable at table level
|
||||
await db_default.create_table(
|
||||
"table_level_enabled",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
storage_options={"new_table_enable_stable_row_ids": "true"},
|
||||
)
|
||||
lance_ds = lance.dataset(tmp_path / "table_level_enabled.lance")
|
||||
fragments = lance_ds.get_fragments()
|
||||
assert len(fragments) > 0
|
||||
assert fragments[0].metadata.row_id_meta is not None, (
|
||||
"Table should have stable row IDs when enabled at table level"
|
||||
)
|
||||
|
||||
# Case 2: Connection has stable row IDs, override with false at table level
|
||||
await db_with_stable.create_table(
|
||||
"table_level_disabled",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
storage_options={"new_table_enable_stable_row_ids": "false"},
|
||||
)
|
||||
lance_ds = lance.dataset(tmp_path / "table_level_disabled.lance")
|
||||
fragments = lance_ds.get_fragments()
|
||||
assert len(fragments) > 0
|
||||
assert fragments[0].metadata.row_id_meta is None, (
|
||||
"Table should NOT have stable row IDs when disabled at table level"
|
||||
)
|
||||
|
||||
|
||||
def test_create_table_stable_row_ids_table_level_override_sync(tmp_path):
|
||||
"""Test that stable_row_ids can be enabled/disabled at create_table level (sync)."""
|
||||
# Connect without any stable row ID setting
|
||||
db_default = lancedb.connect(tmp_path)
|
||||
|
||||
# Connect with stable row IDs enabled at connection level
|
||||
db_with_stable = lancedb.connect(
|
||||
tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"}
|
||||
)
|
||||
|
||||
# Case 1: No connection setting, enable at table level
|
||||
tbl = db_default.create_table(
|
||||
"table_level_enabled_sync",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
storage_options={"new_table_enable_stable_row_ids": "true"},
|
||||
)
|
||||
lance_ds = tbl.to_lance()
|
||||
fragments = lance_ds.get_fragments()
|
||||
assert len(fragments) > 0
|
||||
assert fragments[0].metadata.row_id_meta is not None, (
|
||||
"Table should have stable row IDs when enabled at table level"
|
||||
)
|
||||
|
||||
# Case 2: Connection has stable row IDs, override with false at table level
|
||||
tbl = db_with_stable.create_table(
|
||||
"table_level_disabled_sync",
|
||||
data=[{"id": i} for i in range(10)],
|
||||
storage_options={"new_table_enable_stable_row_ids": "false"},
|
||||
)
|
||||
lance_ds = tbl.to_lance()
|
||||
fragments = lance_ds.get_fragments()
|
||||
assert len(fragments) > 0
|
||||
assert fragments[0].metadata.row_id_meta is None, (
|
||||
"Table should NOT have stable row IDs when disabled at table level"
|
||||
)
|
||||
|
||||
|
||||
def test_open_table_sync(tmp_db: lancedb.DBConnection):
|
||||
tmp_db.create_table("test", data=[{"id": 0}])
|
||||
assert tmp_db.open_table("test").count_rows() == 1
|
||||
|
||||
@@ -105,12 +105,12 @@ test-log = "0.2"
|
||||
|
||||
[features]
|
||||
default = ["aws", "gcs", "azure", "dynamodb", "oss"]
|
||||
aws = ["lance/aws", "lance-io/aws"]
|
||||
oss = ["lance/oss", "lance-io/oss"]
|
||||
gcs = ["lance/gcp", "lance-io/gcp"]
|
||||
azure = ["lance/azure", "lance-io/azure"]
|
||||
aws = ["lance/aws", "lance-io/aws", "lance-namespace-impls/dir-aws"]
|
||||
oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
|
||||
gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
|
||||
azure = ["lance/azure", "lance-io/azure", "lance-namespace-impls/dir-azure"]
|
||||
dynamodb = ["lance/dynamodb", "aws"]
|
||||
remote = ["dep:reqwest", "dep:http"]
|
||||
remote = ["dep:reqwest", "dep:http", "lance-namespace-impls/rest"]
|
||||
fp16kernels = ["lance-linalg/fp16kernels"]
|
||||
s3-test = []
|
||||
bedrock = ["dep:aws-sdk-bedrockruntime"]
|
||||
|
||||
@@ -35,6 +35,7 @@ pub const LANCE_FILE_EXTENSION: &str = "lance";
|
||||
|
||||
pub const OPT_NEW_TABLE_STORAGE_VERSION: &str = "new_table_data_storage_version";
|
||||
pub const OPT_NEW_TABLE_V2_MANIFEST_PATHS: &str = "new_table_enable_v2_manifest_paths";
|
||||
pub const OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS: &str = "new_table_enable_stable_row_ids";
|
||||
|
||||
/// Controls how new tables should be created
|
||||
#[derive(Clone, Debug, Default)]
|
||||
@@ -48,6 +49,12 @@ pub struct NewTableConfig {
|
||||
/// V2 manifest paths are more efficient than V2 manifest paths but are not
|
||||
/// supported by old clients.
|
||||
pub enable_v2_manifest_paths: Option<bool>,
|
||||
/// Whether to enable stable row IDs for new tables
|
||||
///
|
||||
/// When enabled, row IDs remain stable after compaction, update, delete,
|
||||
/// and merges. This is useful for materialized views and other use cases
|
||||
/// that need to track source rows across these operations.
|
||||
pub enable_stable_row_ids: Option<bool>,
|
||||
}
|
||||
|
||||
/// Options specific to the listing database
|
||||
@@ -87,6 +94,14 @@ impl ListingDatabaseOptions {
|
||||
})
|
||||
})
|
||||
.transpose()?,
|
||||
enable_stable_row_ids: map
|
||||
.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS)
|
||||
.map(|s| {
|
||||
s.parse::<bool>().map_err(|_| Error::InvalidInput {
|
||||
message: format!("enable_stable_row_ids must be a boolean, received {}", s),
|
||||
})
|
||||
})
|
||||
.transpose()?,
|
||||
};
|
||||
// We just assume that any options that are not new table config options are storage options
|
||||
let storage_options = map
|
||||
@@ -94,6 +109,7 @@ impl ListingDatabaseOptions {
|
||||
.filter(|(key, _)| {
|
||||
key.as_str() != OPT_NEW_TABLE_STORAGE_VERSION
|
||||
&& key.as_str() != OPT_NEW_TABLE_V2_MANIFEST_PATHS
|
||||
&& key.as_str() != OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS
|
||||
})
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect();
|
||||
@@ -118,6 +134,12 @@ impl DatabaseOptions for ListingDatabaseOptions {
|
||||
enable_v2_manifest_paths.to_string(),
|
||||
);
|
||||
}
|
||||
if let Some(enable_stable_row_ids) = self.new_table_config.enable_stable_row_ids {
|
||||
map.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
enable_stable_row_ids.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -497,7 +519,7 @@ impl ListingDatabase {
|
||||
fn extract_storage_overrides(
|
||||
&self,
|
||||
request: &CreateTableRequest,
|
||||
) -> Result<(Option<LanceFileVersion>, Option<bool>)> {
|
||||
) -> Result<(Option<LanceFileVersion>, Option<bool>, Option<bool>)> {
|
||||
let storage_options = request
|
||||
.write_options
|
||||
.lance_write_params
|
||||
@@ -518,7 +540,19 @@ impl ListingDatabase {
|
||||
message: "enable_v2_manifest_paths must be a boolean".to_string(),
|
||||
})?;
|
||||
|
||||
Ok((storage_version_override, v2_manifest_override))
|
||||
let stable_row_ids_override = storage_options
|
||||
.and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
|
||||
.map(|s| s.parse::<bool>())
|
||||
.transpose()
|
||||
.map_err(|_| Error::InvalidInput {
|
||||
message: "enable_stable_row_ids must be a boolean".to_string(),
|
||||
})?;
|
||||
|
||||
Ok((
|
||||
storage_version_override,
|
||||
v2_manifest_override,
|
||||
stable_row_ids_override,
|
||||
))
|
||||
}
|
||||
|
||||
/// Prepare write parameters for table creation
|
||||
@@ -527,6 +561,7 @@ impl ListingDatabase {
|
||||
request: &CreateTableRequest,
|
||||
storage_version_override: Option<LanceFileVersion>,
|
||||
v2_manifest_override: Option<bool>,
|
||||
stable_row_ids_override: Option<bool>,
|
||||
) -> lance::dataset::WriteParams {
|
||||
let mut write_params = request
|
||||
.write_options
|
||||
@@ -571,6 +606,13 @@ impl ListingDatabase {
|
||||
write_params.enable_v2_manifest_paths = enable_v2_manifest_paths;
|
||||
}
|
||||
|
||||
// Apply enable_stable_row_ids: table-level override takes precedence over connection config
|
||||
if let Some(enable_stable_row_ids) =
|
||||
stable_row_ids_override.or(self.new_table_config.enable_stable_row_ids)
|
||||
{
|
||||
write_params.enable_stable_row_ids = enable_stable_row_ids;
|
||||
}
|
||||
|
||||
if matches!(&request.mode, CreateTableMode::Overwrite) {
|
||||
write_params.mode = WriteMode::Overwrite;
|
||||
}
|
||||
@@ -706,11 +748,15 @@ impl Database for ListingDatabase {
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.table_uri(&request.name).unwrap());
|
||||
|
||||
let (storage_version_override, v2_manifest_override) =
|
||||
let (storage_version_override, v2_manifest_override, stable_row_ids_override) =
|
||||
self.extract_storage_overrides(&request)?;
|
||||
|
||||
let write_params =
|
||||
self.prepare_write_params(&request, storage_version_override, v2_manifest_override);
|
||||
let write_params = self.prepare_write_params(
|
||||
&request,
|
||||
storage_version_override,
|
||||
v2_manifest_override,
|
||||
stable_row_ids_override,
|
||||
);
|
||||
|
||||
let data_schema = request.data.arrow_schema();
|
||||
|
||||
@@ -921,7 +967,7 @@ impl Database for ListingDatabase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::connection::ConnectRequest;
|
||||
use crate::database::{CreateTableData, CreateTableMode, CreateTableRequest};
|
||||
use crate::database::{CreateTableData, CreateTableMode, CreateTableRequest, WriteOptions};
|
||||
use crate::table::{Table, TableDefinition};
|
||||
use arrow_array::{Int32Array, RecordBatch, StringArray};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
@@ -1621,4 +1667,267 @@ mod tests {
|
||||
// Cloned table should have all 8 rows from the latest version
|
||||
assert_eq!(cloned_table.count_rows(None).await.unwrap(), 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_table_with_stable_row_ids_connection_level() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
|
||||
// Create database with stable row IDs enabled at connection level
|
||||
let mut options = HashMap::new();
|
||||
options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"true".to_string(),
|
||||
);
|
||||
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options,
|
||||
read_consistency_interval: None,
|
||||
session: None,
|
||||
};
|
||||
|
||||
let db = ListingDatabase::connect_with_options(&request)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify the config was parsed correctly
|
||||
assert_eq!(db.new_table_config.enable_stable_row_ids, Some(true));
|
||||
|
||||
// Create a table - it should inherit the stable row IDs setting
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||
vec![Ok(batch)],
|
||||
schema.clone(),
|
||||
));
|
||||
|
||||
let table = db
|
||||
.create_table(CreateTableRequest {
|
||||
name: "test_stable".to_string(),
|
||||
namespace: vec![],
|
||||
data: CreateTableData::Data(reader),
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify table was created successfully
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_table_with_stable_row_ids_table_level() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
|
||||
// Verify connection has no stable row IDs config
|
||||
assert_eq!(db.new_table_config.enable_stable_row_ids, None);
|
||||
|
||||
// Create a table with stable row IDs enabled at table level via storage_options
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||
vec![Ok(batch)],
|
||||
schema.clone(),
|
||||
));
|
||||
|
||||
let mut storage_options = HashMap::new();
|
||||
storage_options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"true".to_string(),
|
||||
);
|
||||
|
||||
let write_options = WriteOptions {
|
||||
lance_write_params: Some(lance::dataset::WriteParams {
|
||||
store_params: Some(lance::io::ObjectStoreParams {
|
||||
storage_options: Some(storage_options),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
|
||||
let table = db
|
||||
.create_table(CreateTableRequest {
|
||||
name: "test_stable_table_level".to_string(),
|
||||
namespace: vec![],
|
||||
data: CreateTableData::Data(reader),
|
||||
mode: CreateTableMode::Create,
|
||||
write_options,
|
||||
location: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify table was created successfully
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_create_table_stable_row_ids_table_overrides_connection() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
|
||||
// Create database with stable row IDs enabled at connection level
|
||||
let mut options = HashMap::new();
|
||||
options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"true".to_string(),
|
||||
);
|
||||
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options,
|
||||
read_consistency_interval: None,
|
||||
session: None,
|
||||
};
|
||||
|
||||
let db = ListingDatabase::connect_with_options(&request)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(db.new_table_config.enable_stable_row_ids, Some(true));
|
||||
|
||||
// Create table with stable row IDs disabled at table level (overrides connection)
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let reader = Box::new(arrow_array::RecordBatchIterator::new(
|
||||
vec![Ok(batch)],
|
||||
schema.clone(),
|
||||
));
|
||||
|
||||
let mut storage_options = HashMap::new();
|
||||
storage_options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"false".to_string(),
|
||||
);
|
||||
|
||||
let write_options = WriteOptions {
|
||||
lance_write_params: Some(lance::dataset::WriteParams {
|
||||
store_params: Some(lance::io::ObjectStoreParams {
|
||||
storage_options: Some(storage_options),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
|
||||
let table = db
|
||||
.create_table(CreateTableRequest {
|
||||
name: "test_override".to_string(),
|
||||
namespace: vec![],
|
||||
data: CreateTableData::Data(reader),
|
||||
mode: CreateTableMode::Create,
|
||||
write_options,
|
||||
location: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Verify table was created successfully
|
||||
assert_eq!(table.count_rows(None).await.unwrap(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_stable_row_ids_invalid_value() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
|
||||
// Try to create database with invalid stable row IDs value
|
||||
let mut options = HashMap::new();
|
||||
options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"not_a_boolean".to_string(),
|
||||
);
|
||||
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options,
|
||||
read_consistency_interval: None,
|
||||
session: None,
|
||||
};
|
||||
|
||||
let result = ListingDatabase::connect_with_options(&request).await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(matches!(
|
||||
result.unwrap_err(),
|
||||
Error::InvalidInput { message } if message.contains("enable_stable_row_ids must be a boolean")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stable_row_ids_config_serialization() {
|
||||
// Test that ListingDatabaseOptions correctly serializes stable_row_ids
|
||||
let mut options = HashMap::new();
|
||||
options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"true".to_string(),
|
||||
);
|
||||
|
||||
// Parse the options
|
||||
let db_options = ListingDatabaseOptions::parse_from_map(&options).unwrap();
|
||||
assert_eq!(
|
||||
db_options.new_table_config.enable_stable_row_ids,
|
||||
Some(true)
|
||||
);
|
||||
|
||||
// Serialize back to map
|
||||
let mut serialized = HashMap::new();
|
||||
db_options.serialize_into_map(&mut serialized);
|
||||
|
||||
assert_eq!(
|
||||
serialized.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS),
|
||||
Some(&"true".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stable_row_ids_config_parse_false() {
|
||||
let mut options = HashMap::new();
|
||||
options.insert(
|
||||
OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(),
|
||||
"false".to_string(),
|
||||
);
|
||||
|
||||
let db_options = ListingDatabaseOptions::parse_from_map(&options).unwrap();
|
||||
assert_eq!(
|
||||
db_options.new_table_config.enable_stable_row_ids,
|
||||
Some(false)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stable_row_ids_config_not_set() {
|
||||
let options = HashMap::new();
|
||||
|
||||
let db_options = ListingDatabaseOptions::parse_from_map(&options).unwrap();
|
||||
assert_eq!(db_options.new_table_config.enable_stable_row_ids, None);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user