rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-18 12:08:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: update lance dependency to v10.0.0

Browse Source
This commit is contained in:
Xuanwo
2026-08-10 14:23:29 +08:00
parent 12405a4077
commit 7c7efa9743
19 changed files with 381 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+162 -134
View File
@@ -775,7 +775,7 @@ dependencies = [
"http 0.2.12",
"http 1.5.0",
"http-body 1.1.0",
"lru 0.16.4",
"lru",
"percent-encoding",
"regex-lite",
"sha2 0.11.0",
@@ -1970,6 +1970,15 @@ dependencies = [
"spin 0.10.1",
]
[[package]]
name = "crc32c"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
dependencies = [
"rustc_version",
]
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -3441,12 +3450,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "frostem"
version = "1.20260804.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82eb03a32a1d50555353c85a7b9d3279a6f1e91af9890b789acdf544ed57c8d7"
[[package]]
name = "fs_extra"
version = "1.3.0"
@@ -3455,8 +3458,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0981ce90521824089f3cd68a48b1c4c89e9ad96d0eb74f58d6e550a3d604545"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -3813,23 +3817,14 @@ dependencies = [
[[package]]
name = "goosefs-sdk"
version = "0.1.9"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5"
checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
dependencies = [
"arc-swap",
"async-trait",
"bytes",
"dashmap",
"fastrand",
"futures",
"hostname",
"io-uring",
"itoa",
"libc",
"lru 0.18.2",
"memmap2 0.9.10",
"moka",
"prost",
"prost-types",
"rand 0.9.5",
@@ -3842,7 +3837,6 @@ dependencies = [
"tonic-prost",
"tracing",
"uuid",
"xxhash-rust",
]
[[package]]
@@ -4815,8 +4809,9 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830548f9fc92ae74b848a504282d4da06f016f2ee94b663773c2738b9cb61c42"
dependencies = [
"arc-swap",
"arrow",
@@ -4890,8 +4885,9 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24187f972374567bb3573cffd8728f2f4f7f06d644c3c5d4430b79d6b0b67300"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4913,7 +4909,8 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4927,17 +4924,20 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f"
dependencies = [
"arrow-array",
"arrow-schema",
"half",
"lance-arrow-scalar",
]
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55fa50ad941e25298afb54eec107c3584f80c16db4adf7a6e4e9c4b6066f4281"
dependencies = [
"arrayref",
"crunchy",
@@ -4947,8 +4947,9 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f4fd872bfe948150a6878d983327c4f150dc1b629e94ae7677310fa6a3f35f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4988,8 +4989,9 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "230770734cd5f6fe1f1cb4a66750acad5c5a862c23c7f587d7def57c9f2c5f6b"
dependencies = [
"arrow",
"arrow-array",
@@ -5019,8 +5021,9 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0c9edcea9dbf154a3baf470595bdcdb7ef6b44dcf2b5a0c28e8815c2d4837c5"
dependencies = [
"arrow",
"arrow-array",
@@ -5037,8 +5040,9 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03ac280ef94d66c2e7a4b0104e60d548f45156f114f02cd0ac57423721ec96ec"
dependencies = [
"proc-macro2",
"quote",
@@ -5047,8 +5051,9 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65b345fecf1792d4147982e9ded1cf211992a442bcf73dd17a598b1c8c9b53a5"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5074,6 +5079,7 @@ dependencies = [
"prost",
"prost-build",
"rand 0.9.5",
"strum 0.26.3",
"tokio",
"tracing",
"xxhash-rust",
@@ -5082,8 +5088,9 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "203400160ecd4ac6f67f6bfdb3c186f94758ff70a6ca76d4f300c327f5c14a11"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5114,8 +5121,9 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db67792e22332a7be08d35e60ce9e4c6fb7f75f70ab1566b67b462d1aac21b13"
dependencies = [
"arc-swap",
"arrow",
@@ -5182,8 +5190,9 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49b5700d71f43e5da2057908f3243a6bc01247b5dd12247313f789c280155aae"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5205,8 +5214,9 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b8a6872b9bff95e75e223e04011feebd990d8784878eaf5df73c86d6acebb47"
dependencies = [
"arrow",
"arrow-array",
@@ -5218,6 +5228,7 @@ dependencies = [
"bytes",
"chrono",
"futures",
"goosefs-sdk",
"http 1.5.0",
"io-uring",
"lance-arrow",
@@ -5242,8 +5253,9 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a1dd49fdf61da86650dc335fbe1f239803f9d73bc2bc5ffbc0c905dd8809892"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5259,8 +5271,9 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1449221f599147e570bec6e09eaea6ce6b9286f632e00a27f945dd0c6b70ccd8"
dependencies = [
"arrow",
"async-trait",
@@ -5272,8 +5285,9 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c30bff40ab9a9a58313818ab97241cff7bab2b90b8d65039908425a065a797de"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5303,6 +5317,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"time",
"tokio",
"tower",
"tower-http 0.5.2",
@@ -5326,8 +5341,9 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c66c5cb12ed194c7c8e427a0e90e990cdbc7159edba2bfc7b7e8db2c90c747af"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5342,8 +5358,9 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cd2a39f2e288d1acecf887cb9cc655711fb67bcf3b24a2034158dbac7a6f60e"
dependencies = [
"arrow",
"arrow-array",
@@ -5353,7 +5370,6 @@ dependencies = [
"async-trait",
"aws-credential-types",
"aws-sdk-dynamodb",
"blake3",
"byteorder",
"bytes",
"chrono",
@@ -5383,8 +5399,9 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0847352699e869a465b49bd49ecbc467bc5a3dcb168757bf39368362aad05f99"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5397,13 +5414,14 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1"
version = "10.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "254c1e9287789c8d744f93f2010a8389a5ea3fcedc0dddf7649c4117ec830c57"
dependencies = [
"frostem",
"icu_segmenter",
"jieba-rs",
"lindera",
"rust-stemmers",
"serde",
"stop-words",
"unicode-normalization",
@@ -5681,7 +5699,7 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml_ng",
"strum",
"strum 0.28.0",
"strum_macros 0.28.0",
"unicode-blocks",
"unicode-normalization",
@@ -5711,7 +5729,7 @@ dependencies = [
"rkyv",
"serde",
"serde_json",
"strum",
"strum 0.28.0",
"strum_macros 0.28.0",
"thiserror 2.0.18",
]
@@ -5783,15 +5801,6 @@ dependencies = [
"hashbrown 0.16.1",
]
[[package]]
name = "lru"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a"
dependencies = [
"hashbrown 0.17.1",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
@@ -6429,9 +6438,9 @@ dependencies = [
[[package]]
name = "object_store_opendal"
version = "0.58.0"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac"
checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8"
dependencies = [
"async-trait",
"bytes",
@@ -6492,13 +6501,12 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "opendal"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d"
checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1"
dependencies = [
"ctor 1.0.12",
"opendal-core",
"opendal-http-transport-reqwest",
"opendal-layer-concurrent-limit",
"opendal-layer-logging",
"opendal-layer-retry",
@@ -6515,22 +6523,24 @@ dependencies = [
[[package]]
name = "opendal-core"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed"
checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309"
dependencies = [
"anyhow",
"base64 0.23.1",
"base64 0.22.1",
"bytes",
"futures",
"http 1.5.0",
"http-body 1.1.0",
"jiff",
"log",
"md-5 0.11.0",
"mea",
"percent-encoding",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-core",
"reqwest 0.13.4",
"serde",
"serde_json",
"tokio",
@@ -6539,25 +6549,11 @@ dependencies = [
"web-time",
]
[[package]]
name = "opendal-http-transport-reqwest"
version = "0.58.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef"
dependencies = [
"bytes",
"futures",
"http 1.5.0",
"http-body 1.1.0",
"opendal-core",
"reqwest 0.13.4",
]
[[package]]
name = "opendal-layer-concurrent-limit"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081"
checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f"
dependencies = [
"futures",
"http 1.5.0",
@@ -6567,9 +6563,9 @@ dependencies = [
[[package]]
name = "opendal-layer-logging"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c"
checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2"
dependencies = [
"log",
"opendal-core",
@@ -6577,9 +6573,9 @@ dependencies = [
[[package]]
name = "opendal-layer-retry"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d"
checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7"
dependencies = [
"backon",
"log",
@@ -6588,9 +6584,9 @@ dependencies = [
[[package]]
name = "opendal-layer-timeout"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66"
checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3"
dependencies = [
"opendal-core",
"tokio",
@@ -6598,17 +6594,17 @@ dependencies = [
[[package]]
name = "opendal-service-azblob"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767"
checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a"
dependencies = [
"base64 0.23.1",
"base64 0.22.1",
"bytes",
"http 1.5.0",
"log",
"opendal-core",
"opendal-service-azure-common",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-azure-storage",
"reqsign-core",
"reqsign-file-read-tokio",
@@ -6619,18 +6615,17 @@ dependencies = [
[[package]]
name = "opendal-service-azdls"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53"
checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361"
dependencies = [
"base64 0.23.1",
"base64 0.22.1",
"bytes",
"http 1.5.0",
"log",
"mea",
"opendal-core",
"opendal-service-azure-common",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-azure-storage",
"reqsign-core",
"reqsign-file-read-tokio",
@@ -6640,9 +6635,9 @@ dependencies = [
[[package]]
name = "opendal-service-azure-common"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df"
checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051"
dependencies = [
"http 1.5.0",
"opendal-core",
@@ -6650,15 +6645,15 @@ dependencies = [
[[package]]
name = "opendal-service-cos"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0"
checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
dependencies = [
"bytes",
"http 1.5.0",
"log",
"opendal-core",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-core",
"reqsign-file-read-tokio",
"reqsign-tencent-cos",
@@ -6667,9 +6662,9 @@ dependencies = [
[[package]]
name = "opendal-service-gcs"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6"
checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47"
dependencies = [
"async-trait",
"bytes",
@@ -6677,7 +6672,7 @@ dependencies = [
"log",
"opendal-core",
"percent-encoding",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-core",
"reqsign-file-read-tokio",
"reqsign-google",
@@ -6688,9 +6683,9 @@ dependencies = [
[[package]]
name = "opendal-service-goosefs"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78"
checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
dependencies = [
"bytes",
"goosefs-sdk",
@@ -6702,9 +6697,9 @@ dependencies = [
[[package]]
name = "opendal-service-hf"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7"
checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848"
dependencies = [
"bytes",
"hf-xet",
@@ -6712,21 +6707,22 @@ dependencies = [
"log",
"opendal-core",
"percent-encoding",
"reqwest 0.13.4",
"serde",
"serde_json",
]
[[package]]
name = "opendal-service-oss"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b"
checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace"
dependencies = [
"bytes",
"http 1.5.0",
"log",
"opendal-core",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-aliyun-oss",
"reqsign-core",
"reqsign-file-read-tokio",
@@ -6735,18 +6731,18 @@ dependencies = [
[[package]]
name = "opendal-service-s3"
version = "0.58.1"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663"
checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb"
dependencies = [
"base64 0.23.1",
"base64 0.22.1",
"bytes",
"crc-fast",
"crc32c",
"http 1.5.0",
"log",
"md-5 0.11.0",
"opendal-core",
"quick-xml 0.41.0",
"quick-xml 0.39.4",
"reqsign-aws-v4",
"reqsign-core",
"reqsign-file-read-tokio",
@@ -8679,6 +8675,16 @@ dependencies = [
"ordered-multimap",
]
[[package]]
name = "rust-stemmers"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
dependencies = [
"serde",
"serde_derive",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
@@ -9552,6 +9558,15 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "strum"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
dependencies = [
"strum_macros 0.26.4",
]
[[package]]
name = "strum"
version = "0.28.0"
@@ -9574,6 +9589,19 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "strum_macros"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"rustversion",
"syn 2.0.117",
]
[[package]]
name = "strum_macros"
version = "0.28.0"
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=10.0.0", default-features = false }
lance-core = "=10.0.0"
lance-datagen = "=10.0.0"
lance-file = "=10.0.0"
lance-io = { "version" = "=10.0.0", default-features = false }
lance-index = "=10.0.0"
lance-linalg = "=10.0.0"
lance-namespace = "=10.0.0"
lance-namespace-impls = { "version" = "=10.0.0", default-features = false }
lance-table = "=10.0.0"
lance-testing = "=10.0.0"
lance-datafusion = "=10.0.0"
lance-encoding = "=10.0.0"
lance-arrow = "=10.0.0"
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
docs/src/js/classes/Table.md
+3 -9
View File
@@ -431,10 +431,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always
reports the concrete list resolved when the spec was set — `undefined`
never round-trips.
The returned spec — including its `maintainedIndexes` and
`writerConfigDefaults` — mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec).
#### Returns
@@ -807,11 +806,6 @@ All variants require the table to have an unenforced primary key
([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally
requires it to be the single column being bucketed.
Omitting `maintainedIndexes` maintains every index on the table, resolved
here, failing if one cannot be maintained — name them to install anyway.
Naming them pins an exact set, and a still-building index is rejected
rather than quietly omitted.
#### Parameters
* **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)
docs/src/js/interfaces/LsmWriteSpec.md
+1 -3
View File
@@ -34,9 +34,7 @@ Bucket and identity variants: the sharding column.
optional maintainedIndexes: string[];
```
Indexes the MemWAL keeps up to date. Omit to maintain every supported
index, resolved on install — a snapshot, so indexes created later are not
maintained. Pass `[]` for none.
Names of indexes the MemWAL should keep up to date during writes.
***
java/pom.xml
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>11.0.0-beta.3</lance-core.version>
<lance-core.version>10.0.0</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/lancedb/table.ts
+4 -14
View File
@@ -197,11 +197,7 @@ export interface LsmWriteSpec {
column?: string;
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
numBuckets?: number;
/**
* Indexes the MemWAL keeps up to date. Omit to maintain every supported
* index, resolved on install — a snapshot, so indexes created later are not
* maintained. Pass `[]` for none.
*/
/** Names of indexes the MemWAL should keep up to date during writes. */
maintainedIndexes?: string[];
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
writerConfigDefaults?: Record<string, string>;
@@ -599,11 +595,6 @@ export abstract class Table {
* All variants require the table to have an unenforced primary key
* ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
* requires it to be the single column being bucketed.
*
* Omitting `maintainedIndexes` maintains every index on the table, resolved
* here, failing if one cannot be maintained — name them to install anyway.
* Naming them pins an exact set, and a still-building index is rejected
* rather than quietly omitted.
* @param {LsmWriteSpec} spec The sharding spec to install.
* @returns {Promise<void>}
* @example
@@ -631,10 +622,9 @@ export abstract class Table {
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* The returned spec mirrors what was passed to
* {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
* reports the concrete list resolved when the spec was set — `undefined`
* never round-trips.
* The returned spec — including its `maintainedIndexes` and
* `writerConfigDefaults` — mirrors what was passed to
* {@link Table#setLsmWriteSpec}.
* @returns {Promise<LsmWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
nodejs/src/table.rs
+6 -6
View File
@@ -772,8 +772,7 @@ pub struct LsmWriteSpec {
pub column: Option<String>,
/// Bucket variant: the number of buckets, in `[1, 1024]`.
pub num_buckets: Option<u32>,
/// Indexes the MemWAL keeps up to date. Omitted resolves every
/// maintainable index on install; an empty array means none.
/// Names of indexes the MemWAL should keep up to date during writes.
pub maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
pub writer_config_defaults: Option<HashMap<String, String>>,
@@ -783,6 +782,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
type Error = napi::Error;
fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
let maintained = value.maintained_indexes.unwrap_or_default();
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
let spec = match value.spec_type.as_str() {
"bucket" => {
@@ -809,7 +809,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
};
Ok(spec
.with_maintained_indexes(value.maintained_indexes)
.with_maintained_indexes(maintained)
.with_writer_config_defaults(writer_config_defaults))
}
}
@@ -827,7 +827,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "bucket".to_string(),
column: Some(column),
num_buckets: Some(num_buckets),
maintained_indexes,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Identity {
@@ -838,7 +838,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "identity".to_string(),
column: Some(column),
num_buckets: None,
maintained_indexes,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Unsharded {
@@ -848,7 +848,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "unsharded".to_string(),
column: None,
num_buckets: None,
maintained_indexes,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
}
python/pyproject.toml
+1 -1
View File
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==9.0.0rc1",
"pylance==10.0.0",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
python/python/lancedb/_lancedb.pyi
+4 -7
View File
@@ -653,10 +653,9 @@ class LsmWriteSpec:
def identity(column: str) -> "LsmWriteSpec": ...
@staticmethod
def unsharded() -> "LsmWriteSpec": ...
def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec":
"""Set which indexes the MemWAL keeps up to date. None resolves every
index on the table at install, failing if one cannot be maintained;
a list is verbatim, empty means none."""
def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec":
"""Return a copy of this spec asking the MemWAL to keep the named
indexes up to date as rows are appended."""
...
def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec":
"""Return a copy of this spec recording the given default
@@ -671,9 +670,7 @@ class LsmWriteSpec:
@property
def num_buckets(self) -> Optional[int]: ...
@property
def maintained_indexes(self) -> Optional[List[str]]:
"""Indexes the MemWAL keeps up to date, or None for every supported one."""
...
def maintained_indexes(self) -> List[str]: ...
@property
def writer_config_defaults(self) -> Dict[str, str]: ...
python/python/lancedb/table.py
+3 -10
View File
@@ -4676,13 +4676,6 @@ class AsyncTable:
via [`set_unenforced_primary_key`]; bucket sharding additionally
requires it to be the single column being bucketed.
By default the MemWAL maintains every index on the table, resolved
here — a snapshot, so an index created afterwards needs the spec unset
and set again. This fails if one cannot be maintained; name the set
with ``with_maintained_indexes`` to install anyway. That pins an exact
set (a still-building index is rejected, not omitted); ``[]`` maintains
none.
Parameters
----------
spec : LsmWriteSpec
@@ -4709,9 +4702,9 @@ class AsyncTable:
Returns ``None`` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with `unset_lsm_write_spec`).
The returned spec mirrors what was passed to `set_lsm_write_spec`,
except that ``maintained_indexes`` always reports the concrete list
resolved when the spec was set — ``None`` never round-trips.
The returned spec — including its ``maintained_indexes`` and
``writer_config_defaults`` — mirrors what was passed to
`set_lsm_write_spec`.
"""
return await self._inner.get_lsm_write_spec()
python/python/tests/test_lsm_write_spec.py
+4 -11
View File
@@ -83,9 +83,7 @@ def test_lsm_write_spec_repr():
assert s.spec_type == "bucket"
assert s.column == "id"
assert s.num_buckets == 4
# A fresh spec defers its maintained set to install time.
assert s.maintained_indexes is None
assert s.with_maintained_indexes([]).maintained_indexes == []
assert s.maintained_indexes == []
assert "bucket" in repr(s)
assert "id" in repr(s)
assert "4" in repr(s)
@@ -171,23 +169,18 @@ def test_get_lsm_write_spec(tmp_path):
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema). Leaving the
# maintained set to be inferred picks up the index on the table, so the
# spec reads back naming it rather than as "infer".
# Identity round-trips (column recovered from the schema).
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
assert spec.maintained_indexes == [idx_name]
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column). Opting out is distinct from
# the inferred default.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
# Unsharded round-trips (no routing column).
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
assert spec.column is None
assert spec.maintained_indexes == []
@pytest.mark.asyncio
python/python/tests/test_merge_insert_lsm.py
+2 -2
View File
@@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path):
table.create_index("text", config=FTS())
# No maintained indexes: the active memtable FTS arm cannot serve un-compacted
# docs, so the search would silently omit them — reject instead.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
with pytest.raises(Exception, match="maintained"):
table.search("fox", query_type="fts", fts_columns="text").to_arrow()
@@ -631,7 +631,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path):
)
# Spec with NO maintained indexes: the base vector index's catch-up is untracked,
# so the scanner rejects rather than risk dropping compacted-but-unindexed rows.
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
with pytest.raises(Exception, match="maintained"):
table.search([1.0] * VECTOR_DIM).to_arrow()
python/src/table.rs
+15 -31
View File
@@ -246,22 +246,12 @@ impl From<lancedb::table::MergeResult> for MergeResult {
}
}
/// Render for `__repr__`, so the default reads as Python's `None` rather than
/// Rust's `Some([..])`.
fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
match maintained {
Some(names) => format!("{:?}", names),
None => "None".to_string(),
}
}
/// Specification selecting Lance's MemWAL LSM-style write path for
/// `merge_insert`.
///
/// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
/// classmethods, then optionally chain `with_maintained_indexes(...)` and
/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the
/// MemWAL supports, resolved on install.
/// `with_writer_config_defaults(...)`.
#[pyclass(from_py_object)]
#[derive(Clone, Debug)]
pub struct LsmWriteSpec {
@@ -301,11 +291,11 @@ impl LsmWriteSpec {
}
}
/// Set which indexes the MemWAL maintains. `None` (the default)
/// resolves every supported index on install; a list is verbatim,
/// and an empty list maintains nothing.
#[pyo3(signature = (indexes))]
pub fn with_maintained_indexes(&self, indexes: Option<Vec<String>>) -> Self {
/// Replace the list of indexes the MemWAL should keep up to date as
/// rows are appended. Each name must reference an index that
/// already exists on the table at the time `set_lsm_write_spec`
/// is called.
pub fn with_maintained_indexes(&self, indexes: Vec<String>) -> Self {
Self {
inner: self.inner.clone().with_maintained_indexes(indexes),
}
@@ -327,29 +317,23 @@ impl LsmWriteSpec {
maintained_indexes,
writer_config_defaults,
} => format!(
"LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
column,
num_buckets,
fmt_maintained(maintained_indexes),
writer_config_defaults,
"LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})",
column, num_buckets, maintained_indexes, writer_config_defaults,
),
lancedb::table::LsmWriteSpec::Identity {
column,
maintained_indexes,
writer_config_defaults,
} => format!(
"LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
column,
fmt_maintained(maintained_indexes),
writer_config_defaults,
"LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})",
column, maintained_indexes, writer_config_defaults,
),
lancedb::table::LsmWriteSpec::Unsharded {
maintained_indexes,
writer_config_defaults,
} => format!(
"LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
fmt_maintained(maintained_indexes),
writer_config_defaults,
"LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})",
maintained_indexes, writer_config_defaults,
),
}
}
@@ -384,10 +368,10 @@ impl LsmWriteSpec {
}
}
/// Indexes the MemWAL keeps up to date, or `None` for every supported one.
/// Names of indexes the MemWAL should keep up to date during writes.
#[getter]
pub fn maintained_indexes(&self) -> Option<Vec<String>> {
self.inner.maintained_indexes().map(<[String]>::to_vec)
pub fn maintained_indexes(&self) -> Vec<String> {
self.inner.maintained_indexes().to_vec()
}
/// Default `ShardWriter` configuration recorded by this spec.
python/uv.lock
Generated
+95 -95
View File
@@ -799,7 +799,7 @@ name = "cuda-bindings"
version = "13.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-pathfinder", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "cuda-pathfinder" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" },
@@ -834,37 +834,37 @@ wheels = [
[package.optional-dependencies]
cublas = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cublas" },
]
cudart = [
{ name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-runtime" },
]
cufft = [
{ name = "nvidia-cufft", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cufft" },
]
cufile = [
{ name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cufile" },
]
cupti = [
{ name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-cupti" },
]
curand = [
{ name = "nvidia-curand", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-curand" },
]
cusolver = [
{ name = "nvidia-cusolver", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cusolver" },
]
cusparse = [
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cusparse" },
]
nvjitlink = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-nvjitlink" },
]
nvrtc = [
{ name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc" },
]
nvtx = [
{ name = "nvidia-nvtx", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-nvtx" },
]
[[package]]
@@ -1023,7 +1023,7 @@ name = "exceptiongroup"
version = "1.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
wheels = [
@@ -1440,16 +1440,16 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "cachetools", marker = "python_full_version < '3.11'" },
{ name = "certifi", marker = "python_full_version < '3.11'" },
{ name = "httpx", marker = "python_full_version < '3.11'" },
{ name = "ibm-cos-sdk", marker = "python_full_version < '3.11'" },
{ name = "lomond", marker = "python_full_version < '3.11'" },
{ name = "packaging", marker = "python_full_version < '3.11'" },
{ name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "requests", marker = "python_full_version < '3.11'" },
{ name = "tabulate", marker = "python_full_version < '3.11'" },
{ name = "urllib3", marker = "python_full_version < '3.11'" },
{ name = "cachetools" },
{ name = "certifi" },
{ name = "httpx" },
{ name = "ibm-cos-sdk" },
{ name = "lomond" },
{ name = "packaging" },
{ name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" } },
{ name = "requests" },
{ name = "tabulate" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/c7/56/2e3df38a1f13062095d7bde23c87a92f3898982993a15186b1bfecbd206f/ibm_watsonx_ai-1.3.42.tar.gz", hash = "sha256:ee5be59009004245d957ce97d1227355516df95a2640189749487614fef674ff", size = 688651, upload-time = "2025-10-01T13:35:41.527Z" }
wheels = [
@@ -1468,17 +1468,17 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "cachetools", marker = "python_full_version >= '3.11'" },
{ name = "certifi", marker = "python_full_version >= '3.11'" },
{ name = "httpx", marker = "python_full_version >= '3.11'" },
{ name = "ibm-cos-sdk", marker = "python_full_version >= '3.11'" },
{ name = "lomond", marker = "python_full_version >= '3.11'" },
{ name = "packaging", marker = "python_full_version >= '3.11'" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "cachetools" },
{ name = "certifi" },
{ name = "httpx" },
{ name = "ibm-cos-sdk" },
{ name = "lomond" },
{ name = "packaging" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" },
{ name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
{ name = "requests", marker = "python_full_version >= '3.11'" },
{ name = "tabulate", marker = "python_full_version >= '3.11'" },
{ name = "urllib3", marker = "python_full_version >= '3.11'" },
{ name = "requests" },
{ name = "tabulate" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/29/a3/c756b534696ab2f3f29882fdb7ca7198b7a5c94e10c0a3a327853d6d6b79/ibm_watsonx_ai-1.5.14.tar.gz", hash = "sha256:a756488bd57e87c0fc51be42dcba871143cfe0ac1e805c497c5047e1e4f13e9d", size = 735804, upload-time = "2026-06-22T12:32:43.85Z" }
wheels = [
@@ -1554,17 +1554,17 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" },
{ name = "decorator", marker = "python_full_version < '3.11'" },
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
{ name = "jedi", marker = "python_full_version < '3.11'" },
{ name = "matplotlib-inline", marker = "python_full_version < '3.11'" },
{ name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit", marker = "python_full_version < '3.11'" },
{ name = "pygments", marker = "python_full_version < '3.11'" },
{ name = "stack-data", marker = "python_full_version < '3.11'" },
{ name = "traitlets", marker = "python_full_version < '3.11'" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
{ name = "colorama", marker = "sys_platform == 'win32'" },
{ name = "decorator" },
{ name = "exceptiongroup" },
{ name = "jedi" },
{ name = "matplotlib-inline" },
{ name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit" },
{ name = "pygments" },
{ name = "stack-data" },
{ name = "traitlets" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" }
wheels = [
@@ -1583,18 +1583,18 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" },
{ name = "decorator", marker = "python_full_version >= '3.11'" },
{ name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" },
{ name = "jedi", marker = "python_full_version >= '3.11'" },
{ name = "matplotlib-inline", marker = "python_full_version >= '3.11'" },
{ name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit", marker = "python_full_version >= '3.11'" },
{ name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" },
{ name = "pygments", marker = "python_full_version >= '3.11'" },
{ name = "stack-data", marker = "python_full_version >= '3.11'" },
{ name = "traitlets", marker = "python_full_version >= '3.11'" },
{ name = "typing-extensions", marker = "python_full_version == '3.11.*'" },
{ name = "colorama", marker = "sys_platform == 'win32'" },
{ name = "decorator" },
{ name = "ipython-pygments-lexers" },
{ name = "jedi" },
{ name = "matplotlib-inline" },
{ name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" },
{ name = "prompt-toolkit" },
{ name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" },
{ name = "pygments" },
{ name = "stack-data" },
{ name = "traitlets" },
{ name = "typing-extensions", marker = "python_full_version < '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" }
wheels = [
@@ -1606,7 +1606,7 @@ name = "ipython-pygments-lexers"
version = "1.1.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pygments", marker = "python_full_version >= '3.11'" },
{ name = "pygments" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" }
wheels = [
@@ -2005,7 +2005,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=1.10" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==10.0.0" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2858,7 +2858,7 @@ name = "nvidia-cudnn-cu13"
version = "9.19.0.56"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cublas" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" },
@@ -2870,7 +2870,7 @@ name = "nvidia-cufft"
version = "12.0.0.61"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
@@ -2900,9 +2900,9 @@ name = "nvidia-cusolver"
version = "12.0.4.66"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cublas" },
{ name = "nvidia-cusparse" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
@@ -2914,7 +2914,7 @@ name = "nvidia-cusparse"
version = "12.6.3.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
@@ -3091,10 +3091,10 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "python-dateutil", marker = "python_full_version < '3.11'" },
{ name = "pytz", marker = "python_full_version < '3.11'" },
{ name = "tzdata", marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
{ name = "python-dateutil" },
{ name = "pytz" },
{ name = "tzdata" },
]
sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" }
wheels = [
@@ -3143,11 +3143,11 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" },
{ name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "pytz", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "tzdata", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "python-dateutil" },
{ name = "pytz" },
{ name = "tzdata" },
]
sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" }
wheels = [
@@ -3210,9 +3210,9 @@ resolution-markers = [
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'",
]
dependencies = [
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
{ name = "python-dateutil", marker = "python_full_version >= '3.14'" },
{ name = "tzdata", marker = "(python_full_version >= '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.14' and sys_platform == 'win32')" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } },
{ name = "python-dateutil" },
{ name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" }
wheels = [
@@ -3320,7 +3320,7 @@ name = "pexpect"
version = "4.9.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "ptyprocess", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "ptyprocess" },
]
sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" }
wheels = [
@@ -3912,7 +3912,7 @@ crypto = [
[[package]]
name = "pylance"
version = "7.0.0"
version = "10.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace" },
@@ -3922,12 +3922,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/ac/ad/2f64921bf346e7075aef24a72595db44821724a3d89a9a92dd24e79632aa/pylance-7.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:98422021975be76e72b1572f41b8c9abb3bee5bdc9bfa5e9ce731110a65ed4d1", size = 62134146, upload-time = "2026-05-27T21:59:37.459Z" },
{ url = "https://files.pythonhosted.org/packages/73/1c/c5a01bee0160b55d9a98895cbd33091d038f0a0995b121ab72e629008d02/pylance-7.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bec86ee5b6fbd8bfc493e653f0a1fba0303cfe5492b9b46fc25ab908edc7183", size = 65373684, upload-time = "2026-05-27T22:04:01.584Z" },
{ url = "https://files.pythonhosted.org/packages/eb/da/1fe8b8f7dbfe734d76af76acc994fc360a0d0c79a4874ef69f5a72a58fe3/pylance-7.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881491432c53184e52f8d1db8d5f872f39a03f36fb104bec77b33d379519d8b5", size = 69458555, upload-time = "2026-05-27T22:16:50.567Z" },
{ url = "https://files.pythonhosted.org/packages/76/f0/dd505cf3fd0226ab9d94759acd713125af1d3bfacfd80bbd52e3b9f89509/pylance-7.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18453999e7fff4f76b16d6b7882c9df0628bd142ff95e2461bd7dd5ee3fe0af3", size = 65394430, upload-time = "2026-05-27T22:05:30.923Z" },
{ url = "https://files.pythonhosted.org/packages/17/ba/2357b81034f28eb00790e258ed140289a6a887a7468ca9df6349fd186b27/pylance-7.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:04a58051d408c60fe76d41a220dcaf8fea8fb6d1aa0ca78a709b60bc3cc8d19a", size = 69473470, upload-time = "2026-05-27T22:17:18.935Z" },
{ url = "https://files.pythonhosted.org/packages/1f/ec/5c00b6303a67d787f9475141832cbdc513d674ac3dcaeef8a7b169905e65/pylance-7.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:467d4864af047eaab4e1370e2f1e88e2c6f507c079874421116cb41d78bc3629", size = 74792863, upload-time = "2026-05-27T22:19:23.875Z" },
{ url = "https://files.pythonhosted.org/packages/82/b2/c81de196076c4c8d768f485324a0043e113ce0950a339977d83fee9783ef/pylance-10.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4bba56ae829202b7e9cdc82c92c00a4b52f03e679dc3edfad1db09d1285be2e", size = 69279797, upload-time = "2026-08-07T18:25:24.812Z" },
{ url = "https://files.pythonhosted.org/packages/d2/e9/af671a6225740bd70c1e70bd83fa08091d628b960397cbecc477f16aaaf1/pylance-10.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:489b944827c0271e16a62b4006f8c75acc3bd7fbc381f35388a2afaaae38d438", size = 72775384, upload-time = "2026-08-07T18:31:29.686Z" },
{ url = "https://files.pythonhosted.org/packages/5d/22/e07194195bb3bbdf062b0c31690fc92fccb2686d5e768efa1dc379a93350/pylance-10.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:018efe7d437d326b9049c1223458bd955e85e48509b4b0bbf8b2d3d94075dbff", size = 76632194, upload-time = "2026-08-07T18:44:29.001Z" },
{ url = "https://files.pythonhosted.org/packages/f2/71/ed9956cf657e86a5fee5d5ddcfa7bfed0d7c2968bee39ed85d12629d8c93/pylance-10.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:254bad5d765c14db6c4eddd5323133dee3c76ff6d40c4777afe0b0e91d4c04d2", size = 72804222, upload-time = "2026-08-07T18:31:24.264Z" },
{ url = "https://files.pythonhosted.org/packages/94/96/de449c246b2892df9d5a0af775b79a061e9897e6942bcee5c4ab6d6071f8/pylance-10.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9f0c089e30389d9b7765a4c16f3d9325b05ed4fbf6cc2bbc9418292983472e54", size = 76602015, upload-time = "2026-08-07T18:46:27.724Z" },
{ url = "https://files.pythonhosted.org/packages/53/81/4a5a9072b6d68c4dbb8c7b5530a381c7a6bea4ba92f9eca58ac885142722/pylance-10.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:9fecf46e6835dc64b2d71767d5eab4f73787543a852432e9ba8ecf8527b3cdab", size = 82799247, upload-time = "2026-08-07T18:47:46.855Z" },
]
[[package]]
@@ -4683,10 +4683,10 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "joblib", marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "threadpoolctl", marker = "python_full_version < '3.11'" },
{ name = "joblib" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
{ name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } },
{ name = "threadpoolctl" },
]
sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" }
wheels = [
@@ -4734,13 +4734,13 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "joblib", marker = "python_full_version >= '3.11'" },
{ name = "narwhals", marker = "python_full_version >= '3.11'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "joblib" },
{ name = "narwhals" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },
{ name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "threadpoolctl", marker = "python_full_version >= '3.11'" },
{ name = "threadpoolctl" },
]
sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" }
wheels = [
@@ -4784,7 +4784,7 @@ resolution-markers = [
"python_full_version < '3.11'",
]
dependencies = [
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" }
wheels = [
@@ -4843,7 +4843,7 @@ resolution-markers = [
"python_full_version == '3.11.*'",
]
dependencies = [
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" },
{ name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" }
wheels = [
@@ -4920,7 +4920,7 @@ resolution-markers = [
"python_full_version >= '3.12' and python_full_version < '3.14'",
]
dependencies = [
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } },
]
sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" }
wheels = [
rust/lancedb/src/dataloader/permutation/shuffle.rs
+6 -5
View File
@@ -12,8 +12,7 @@ use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::{
reader::{FileReader, FileReaderOptions},
version::ConcreteFileVersion,
versions,
writer::FileWriterOptions,
writer::{FileWriter, FileWriterOptions},
};
use lance_io::{
ReadBatchParams,
@@ -154,11 +153,13 @@ impl Shuffler {
source: None,
})?;
let object_writer = object_store.create(&path).await?;
let writer = versions::create_writer(
ConcreteFileVersion::V2_1,
let writer = FileWriter::try_new(
object_writer,
schema.clone(),
FileWriterOptions::default(),
FileWriterOptions {
format_version: Some(ConcreteFileVersion::V2_1.into()),
..Default::default()
},
)?;
file_writers.push(writer);
}
rust/lancedb/src/remote/table.rs
+9 -28
View File
@@ -2520,9 +2520,9 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
self.check_mutable().await?;
// Map the spec onto the server's request DTO. `sharding` is internally
// tagged on `mode` to mirror sophon's `Sharding` enum. A null
// `maintained_indexes` asks the server to resolve every maintainable
// index at HEAD; a list is verbatim, an empty one meaning none.
// tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes`
// and `writer_config_defaults` are sent verbatim (an empty list means "no
// maintained indexes", not "default to all").
let sharding = match &spec {
LsmWriteSpec::Bucket {
column,
@@ -5955,18 +5955,17 @@ mod tests {
.await
.unwrap();
// Positions are relative to the first retained token, so dropping the
// leading "hello" stop word does not shift the remaining tokens.
// Lance 10 retains original token positions after stop-word removal.
assert_eq!(
tokens,
vec![
FtsToken {
text: "こんにちは".to_string(),
position: 0,
position: 1,
},
FtsToken {
text: "世界".to_string(),
position: 1,
position: 2,
},
]
);
@@ -6599,7 +6598,7 @@ mod tests {
.unwrap()
});
let spec = crate::table::LsmWriteSpec::unsharded()
.with_maintained_indexes(vec!["id_idx".to_string()])
.with_maintained_indexes(["id_idx"])
.with_writer_config_defaults([("max_memtable_rows", "1000")]);
table.set_lsm_write_spec(spec).await.unwrap();
}
@@ -6618,29 +6617,11 @@ mod tests {
body["sharding"],
serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 })
);
// An unpinned maintained set sends null: resolve server-side.
assert_eq!(body["maintained_indexes"], serde_json::Value::Null);
http::Response::builder().status(200).body("{}").unwrap()
});
table
.set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16))
.await
.unwrap();
}
/// `[]` (none) must stay distinguishable on the wire from null (all).
#[tokio::test]
async fn test_set_lsm_write_spec_no_maintained_indexes() {
let table = Table::new_with_handler("my_table", |request| {
let body = request.body().unwrap().as_bytes().unwrap();
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["maintained_indexes"], serde_json::json!([]));
http::Response::builder().status(200).body("{}").unwrap()
});
table
.set_lsm_write_spec(
crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()),
)
.set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16))
.await
.unwrap();
}
@@ -6719,7 +6700,7 @@ mod tests {
} => {
assert_eq!(column, "id");
assert_eq!(num_buckets, 4);
assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()]));
assert_eq!(maintained_indexes, vec!["id_idx".to_string()]);
assert_eq!(
writer_config_defaults
.get("durable_write")
rust/lancedb/src/table.rs
+41 -171
View File
@@ -370,8 +370,6 @@ pub use self::merge::MergeResult;
/// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default
/// `ShardWriter` configuration recorded in the MemWAL index).
///
/// A fresh spec maintains every index on the table, resolved on install.
///
/// Install a spec with [`Table::set_lsm_write_spec`] and remove it with
/// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch
/// onto the MemWAL writer is a follow-up.
@@ -386,12 +384,9 @@ pub enum LsmWriteSpec {
Bucket {
column: String,
num_buckets: u32,
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Names of indexes (already created on the table) that the
/// MemWAL should maintain in-memory as rows are appended.
maintained_indexes: Vec<String>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
},
@@ -401,41 +396,35 @@ pub enum LsmWriteSpec {
/// distinct value of `column` becomes its own shard.
Identity {
column: String,
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Names of indexes (already created on the table) that the
/// MemWAL should maintain in-memory as rows are appended.
maintained_indexes: Vec<String>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
},
/// No sharding — every `merge_insert` call writes to a single MemWAL shard.
Unsharded {
/// Indexes the MemWAL maintains in-memory as rows are appended.
///
/// `None` means every index it can maintain, resolved on install — a
/// snapshot, so indexes created later need the spec unset and re-set.
/// `Some([])` maintains nothing.
maintained_indexes: Option<Vec<String>>,
/// Names of indexes (already created on the table) that the
/// MemWAL should maintain in-memory as rows are appended.
maintained_indexes: Vec<String>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
writer_config_defaults: HashMap<String, String>,
},
}
impl LsmWriteSpec {
/// Construct a hash-bucket sharding spec maintaining every index on the table.
/// Construct a hash-bucket sharding spec with no maintained indexes.
pub fn bucket(column: impl Into<String>, num_buckets: u32) -> Self {
Self::Bucket {
column: column.into(),
num_buckets,
maintained_indexes: None,
maintained_indexes: Vec::new(),
writer_config_defaults: HashMap::new(),
}
}
/// Construct an identity-sharding spec (shard by the raw value of
/// `column`) maintaining every index on the table.
/// `column`) with no maintained indexes.
///
/// `column` must be a deterministic function of the unenforced primary
/// key: every row with a given primary key must always produce the same
@@ -447,37 +436,28 @@ impl LsmWriteSpec {
pub fn identity(column: impl Into<String>) -> Self {
Self::Identity {
column: column.into(),
maintained_indexes: None,
maintained_indexes: Vec::new(),
writer_config_defaults: HashMap::new(),
}
}
/// Construct an unsharded spec maintaining every index on the table.
/// Construct an unsharded spec with no maintained indexes.
pub fn unsharded() -> Self {
Self::Unsharded {
maintained_indexes: None,
maintained_indexes: Vec::new(),
writer_config_defaults: HashMap::new(),
}
}
/// Set which indexes the MemWAL maintains.
///
/// `None` (the default) resolves to every index on the table at install,
/// failing if one cannot be maintained — name the set to install anyway. A
/// list is verbatim: each name must already exist and be maintainable, and
/// an empty list maintains nothing.
///
/// ```
/// # use lancedb::table::LsmWriteSpec;
/// // Every index the table has when the spec is installed:
/// LsmWriteSpec::unsharded().with_maintained_indexes(None);
/// // Exactly these:
/// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]);
/// // None at all:
/// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new());
/// ```
pub fn with_maintained_indexes(mut self, indexes: impl Into<Option<Vec<String>>>) -> Self {
let indexes = indexes.into();
/// Replace the list of indexes the MemWAL should keep up to date as
/// rows are appended. Each name must reference an index that already
/// exists on the table at the time `set_lsm_write_spec` is called.
pub fn with_maintained_indexes<I, S>(mut self, indexes: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
let v: Vec<String> = indexes.into_iter().map(Into::into).collect();
match &mut self {
Self::Bucket {
maintained_indexes, ..
@@ -487,7 +467,7 @@ impl LsmWriteSpec {
}
| Self::Unsharded {
maintained_indexes, ..
} => *maintained_indexes = indexes,
} => *maintained_indexes = v,
}
self
}
@@ -523,9 +503,8 @@ impl LsmWriteSpec {
self
}
/// Borrow the list of index names this spec asks MemWAL to maintain, or
/// `None` when it asks for every index on the table.
pub fn maintained_indexes(&self) -> Option<&[String]> {
/// Borrow the list of index names this spec asks MemWAL to maintain.
pub fn maintained_indexes(&self) -> &[String] {
match self {
Self::Bucket {
maintained_indexes, ..
@@ -535,7 +514,7 @@ impl LsmWriteSpec {
}
| Self::Unsharded {
maintained_indexes, ..
} => maintained_indexes.as_deref(),
} => maintained_indexes,
}
}
@@ -1733,7 +1712,7 @@ impl Table {
/// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// table
/// .set_lsm_write_spec(
/// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]),
/// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(["id_idx"]),
/// )
/// .await?;
/// # Ok(())
@@ -1755,10 +1734,9 @@ impl Table {
///
/// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no
/// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]).
/// The returned spec mirrors what was passed to
/// [`Table::set_lsm_write_spec`], except that
/// [`LsmWriteSpec::maintained_indexes`] always reports the concrete list
/// resolved when the spec was set — `None` never round-trips.
/// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and
/// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to
/// [`Table::set_lsm_write_spec`].
///
/// # Example
///
@@ -5107,7 +5085,7 @@ mod tests {
// Bucket spec round-trips exactly, including the routing column (recovered
// from its field id), maintained indexes, and writer config defaults.
let spec = LsmWriteSpec::bucket("id", 4)
.with_maintained_indexes(vec![idx_name.clone()])
.with_maintained_indexes([idx_name])
.with_writer_config_defaults([("durable_write", "false")]);
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
@@ -5117,125 +5095,15 @@ mod tests {
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Identity sharding round-trips (column recovered from the schema).
// A spec left at its default maintains every index on the table, so it
// reads back naming the one on the table rather than as "infer".
let spec = LsmWriteSpec::identity("region");
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(
table.get_lsm_write_spec().await.unwrap(),
Some(spec.with_maintained_indexes(vec![idx_name.clone()]))
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
table.unset_lsm_write_spec().await.unwrap();
// Unsharded round-trips (no routing column).
let spec = LsmWriteSpec::unsharded();
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(
table.get_lsm_write_spec().await.unwrap(),
Some(spec.with_maintained_indexes(vec![idx_name]))
);
}
/// The maintained set defaults to every index on the table, resolved at
/// install. An index the memtable cannot build fails the install rather
/// than being dropped: maintaining it would take the table offline for
/// writes, dropping it would hide that from the caller.
#[tokio::test]
async fn test_set_lsm_write_spec_infers_maintained_indexes() {
let tmp_dir = tempdir().unwrap();
let uri = tmp_dir.path().to_str().unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("tag", DataType::Utf8, true),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
],
)
.unwrap();
let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
let conn = ConnectBuilder::new(uri)
.read_consistency_interval(Duration::from_secs(0))
.execute()
.await
.unwrap();
let table = conn.create_table("t", reader).execute().await.unwrap();
table
.create_index(&["id"], Index::BTree(Default::default()))
.name("id_btree".to_string())
.execute()
.await
.unwrap();
table
.create_index(&["tag"], Index::Bitmap(Default::default()))
.name("tag_bitmap".to_string())
.execute()
.await
.unwrap();
// Explicitly naming the bitmap index fails before anything commits.
let err = table
.set_lsm_write_spec(
LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]),
)
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")),
"expected the bitmap index to be rejected, got {err:?}"
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// The default covers every index, so the bitmap fails it too.
let err = table
.set_lsm_write_spec(LsmWriteSpec::unsharded())
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { ref message }
if message.contains("tag_bitmap") && message.contains("maintained_indexes")),
"expected the inferred set to be rejected, got {err:?}"
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Naming the maintainable subset installs.
table
.set_lsm_write_spec(
LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]),
)
.await
.unwrap();
assert_eq!(
table
.get_lsm_write_spec()
.await
.unwrap()
.unwrap()
.maintained_indexes(),
Some(["id_btree".to_string()].as_slice())
);
// Opting out entirely is distinct from the default.
table.unset_lsm_write_spec().await.unwrap();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()))
.await
.unwrap();
assert_eq!(
table
.get_lsm_write_spec()
.await
.unwrap()
.unwrap()
.maintained_indexes(),
Some([].as_slice())
);
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
}
#[tokio::test]
@@ -5340,7 +5208,7 @@ mod tests {
use lance::dataset::WriteDestination;
use lance::dataset::transaction::{DataOverlayGroup, Operation};
use lance_file::version::{ConcreteFileVersion, LanceFileVersion};
use lance_file::writer::FileWriterOptions;
use lance_file::writer::{FileWriter, FileWriterOptions};
use lance_io::utils::CachedFileSize;
use lance_table::format::DataFile;
use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage};
@@ -5411,11 +5279,13 @@ mod tests {
let store = dataset.object_store(None).await.unwrap();
let path = dataset.data_dir().child(filename.clone());
let obj_writer = store.create(&path).await.unwrap();
let mut writer = lance_file::versions::create_writer(
file_version,
let mut writer = FileWriter::try_new(
obj_writer,
overlay_schema,
FileWriterOptions::default(),
FileWriterOptions {
format_version: Some(file_version.into()),
..Default::default()
},
)
.unwrap();
writer
rust/lancedb/src/table/merge.rs
+2 -2
View File
@@ -1161,7 +1161,7 @@ mod lsm_tests {
.unwrap();
let fts_index = table.list_indices().await.unwrap()[0].name.clone();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index]))
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index]))
.await
.unwrap();
@@ -1254,7 +1254,7 @@ mod lsm_tests {
.unwrap();
let vec_index = table.list_indices().await.unwrap()[0].name.clone();
table
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index]))
.set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index]))
.await
.unwrap();
rust/lancedb/src/table/merge/lsm.rs
+8 -74
View File
@@ -29,7 +29,6 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef};
use lance::Dataset;
use lance::dataset::mem_wal::{
DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec,
validate_maintained_indexes,
};
use lance::index::DatasetIndexExt;
use lance_core::datatypes::Schema as LanceSchema;
@@ -38,9 +37,8 @@ use tokio::sync::RwLock;
use uuid::Uuid;
use crate::error::{Error, Result};
use crate::index::IndexConfig;
use crate::table::merge::{MergeInsertBuilder, MergeResult};
use crate::table::{BaseTable, LsmWriteSpec, NativeTable};
use crate::table::{LsmWriteSpec, NativeTable};
/// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`].
/// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`).
@@ -82,44 +80,32 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
}
}
// Before the builder borrows the dataset clone. `list_indices` merges an
// index's segments into one entry, so the result needs no dedup.
let maintained_indexes = {
let dataset = table.dataset.get().await?;
resolve_maintained_indexes(
&dataset,
&table.list_indices().await?,
spec.maintained_indexes(),
)
.await?
};
let mut dataset = (*table.dataset.get().await?).clone();
let mut builder = dataset.initialize_mem_wal();
let writer_config_defaults = match spec {
let (maintained_indexes, writer_config_defaults) = match spec {
LsmWriteSpec::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
..
} => {
builder = builder.bucket_sharding(column, num_buckets);
writer_config_defaults
(maintained_indexes, writer_config_defaults)
}
LsmWriteSpec::Identity {
column,
maintained_indexes,
writer_config_defaults,
..
} => {
builder = builder.identity_sharding(column);
writer_config_defaults
(maintained_indexes, writer_config_defaults)
}
LsmWriteSpec::Unsharded {
maintained_indexes,
writer_config_defaults,
..
} => {
builder = builder.unsharded();
writer_config_defaults
(maintained_indexes, writer_config_defaults)
}
};
builder = builder.maintained_indexes(maintained_indexes);
@@ -131,58 +117,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
Ok(())
}
/// Resolve a spec's maintained-index selection against `indices`, as reported
/// by [`Table::list_indices`](crate::Table::list_indices).
///
/// `None` means every index on the table, snapshotted now. Lance validates
/// either selection against its shard-writer rules, so a spec that installs is
/// one the MemWAL can open.
///
/// An unmaintainable index fails an inferred set rather than being dropped from
/// it — dropping would leave the caller believing it is maintained.
async fn resolve_maintained_indexes(
dataset: &Dataset,
indices: &[IndexConfig],
requested: Option<&[String]>,
) -> Result<Vec<String>> {
let Some(requested) = requested else {
let all: Vec<String> = indices.iter().map(|index| index.name.clone()).collect();
validate_maintained_indexes(dataset, &all)
.await
.map_err(|source| Error::InvalidInput {
message: format!(
"cannot maintain every index on this table: {source}. Set \
maintained_indexes explicitly to choose from {}",
index_name_list(indices),
),
})?;
return Ok(all);
};
for name in requested {
if !indices.iter().any(|index| &index.name == name) {
return Err(Error::InvalidInput {
message: format!(
"maintained index '{}' does not exist on this table; it has {}",
name,
index_name_list(indices),
),
});
}
}
validate_maintained_indexes(dataset, requested).await?;
Ok(requested.to_vec())
}
/// Index names for an error message.
fn index_name_list(indices: &[IndexConfig]) -> String {
if indices.is_empty() {
return "no indexes".to_string();
}
let mut names: Vec<&str> = indices.iter().map(|index| index.name.as_str()).collect();
names.sort_unstable();
format!("[{}]", names.join(", "))
}
// =============================================================================
// unset_lsm_write_spec
// =============================================================================