From 7c7efa9743edbfdf4b5bfbb437bceb327dbbc478 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Mon, 10 Aug 2026 14:23:29 +0800 Subject: [PATCH] feat: update lance dependency to v10.0.0 --- Cargo.lock | 296 ++++++++++-------- Cargo.toml | 28 +- docs/src/js/classes/Table.md | 12 +- docs/src/js/interfaces/LsmWriteSpec.md | 4 +- java/pom.xml | 2 +- nodejs/lancedb/table.ts | 18 +- nodejs/src/table.rs | 12 +- python/pyproject.toml | 2 +- python/python/lancedb/_lancedb.pyi | 11 +- python/python/lancedb/table.py | 13 +- python/python/tests/test_lsm_write_spec.py | 15 +- python/python/tests/test_merge_insert_lsm.py | 4 +- python/src/table.rs | 46 +-- python/uv.lock | 190 +++++------ .../src/dataloader/permutation/shuffle.rs | 11 +- rust/lancedb/src/remote/table.rs | 37 +-- rust/lancedb/src/table.rs | 212 +++---------- rust/lancedb/src/table/merge.rs | 4 +- rust/lancedb/src/table/merge/lsm.rs | 82 +---- 19 files changed, 381 insertions(+), 618 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec6b7cbfb..8cbfecc11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,7 +775,7 @@ dependencies = [ "http 0.2.12", "http 1.5.0", "http-body 1.1.0", - "lru 0.16.4", + "lru", "percent-encoding", "regex-lite", "sha2 0.11.0", @@ -1970,6 +1970,15 @@ dependencies = [ "spin 0.10.1", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -3441,12 +3450,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "frostem" -version = "1.20260804.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82eb03a32a1d50555353c85a7b9d3279a6f1e91af9890b789acdf544ed57c8d7" - [[package]] name = "fs_extra" version = "1.3.0" @@ -3455,8 +3458,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0981ce90521824089f3cd68a48b1c4c89e9ad96d0eb74f58d6e550a3d604545" dependencies = [ "arrow-array", "rand 0.9.5", @@ -3813,23 +3817,14 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.9" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" +checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" dependencies = [ - "arc-swap", "async-trait", "bytes", "dashmap", - "fastrand", - "futures", "hostname", - "io-uring", - "itoa", - "libc", - "lru 0.18.2", - "memmap2 0.9.10", - "moka", "prost", "prost-types", "rand 0.9.5", @@ -3842,7 +3837,6 @@ dependencies = [ "tonic-prost", "tracing", "uuid", - "xxhash-rust", ] [[package]] @@ -4815,8 +4809,9 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830548f9fc92ae74b848a504282d4da06f016f2ee94b663773c2738b9cb61c42" dependencies = [ "arc-swap", "arrow", @@ -4890,8 +4885,9 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24187f972374567bb3573cffd8728f2f4f7f06d644c3c5d4430b79d6b0b67300" dependencies = [ "arrow-array", "arrow-buffer", @@ -4913,7 +4909,8 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79" dependencies = [ "arrow-array", "arrow-buffer", @@ -4927,17 +4924,20 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f" dependencies = [ "arrow-array", "arrow-schema", + "half", "lance-arrow-scalar", ] [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55fa50ad941e25298afb54eec107c3584f80c16db4adf7a6e4e9c4b6066f4281" dependencies = [ "arrayref", "crunchy", @@ -4947,8 +4947,9 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21f4fd872bfe948150a6878d983327c4f150dc1b629e94ae7677310fa6a3f35f" dependencies = [ "arrow-array", "arrow-buffer", @@ -4988,8 +4989,9 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230770734cd5f6fe1f1cb4a66750acad5c5a862c23c7f587d7def57c9f2c5f6b" dependencies = [ "arrow", "arrow-array", @@ -5019,8 +5021,9 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0c9edcea9dbf154a3baf470595bdcdb7ef6b44dcf2b5a0c28e8815c2d4837c5" dependencies = [ "arrow", "arrow-array", @@ -5037,8 +5040,9 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ac280ef94d66c2e7a4b0104e60d548f45156f114f02cd0ac57423721ec96ec" dependencies = [ "proc-macro2", "quote", @@ -5047,8 +5051,9 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b345fecf1792d4147982e9ded1cf211992a442bcf73dd17a598b1c8c9b53a5" dependencies = [ "arrow-arith", "arrow-array", @@ -5074,6 +5079,7 @@ dependencies = [ "prost", "prost-build", "rand 0.9.5", + "strum 0.26.3", "tokio", "tracing", "xxhash-rust", @@ -5082,8 +5088,9 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203400160ecd4ac6f67f6bfdb3c186f94758ff70a6ca76d4f300c327f5c14a11" dependencies = [ "arrow-arith", "arrow-array", @@ -5114,8 +5121,9 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db67792e22332a7be08d35e60ce9e4c6fb7f75f70ab1566b67b462d1aac21b13" dependencies = [ "arc-swap", "arrow", @@ -5182,8 +5190,9 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b5700d71f43e5da2057908f3243a6bc01247b5dd12247313f789c280155aae" dependencies = [ "arrow-array", "arrow-schema", @@ -5205,8 +5214,9 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8a6872b9bff95e75e223e04011feebd990d8784878eaf5df73c86d6acebb47" dependencies = [ "arrow", "arrow-array", @@ -5218,6 +5228,7 @@ dependencies = [ "bytes", "chrono", "futures", + "goosefs-sdk", "http 1.5.0", "io-uring", "lance-arrow", @@ -5242,8 +5253,9 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a1dd49fdf61da86650dc335fbe1f239803f9d73bc2bc5ffbc0c905dd8809892" dependencies = [ "arrow-array", "arrow-buffer", @@ -5259,8 +5271,9 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1449221f599147e570bec6e09eaea6ce6b9286f632e00a27f945dd0c6b70ccd8" dependencies = [ "arrow", "async-trait", @@ -5272,8 +5285,9 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30bff40ab9a9a58313818ab97241cff7bab2b90b8d65039908425a065a797de" dependencies = [ "arrow", "arrow-ipc", @@ -5303,6 +5317,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", + "time", "tokio", "tower", "tower-http 0.5.2", @@ -5326,8 +5341,9 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66c5cb12ed194c7c8e427a0e90e990cdbc7159edba2bfc7b7e8db2c90c747af" dependencies = [ "arrow-array", "arrow-buffer", @@ -5342,8 +5358,9 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cd2a39f2e288d1acecf887cb9cc655711fb67bcf3b24a2034158dbac7a6f60e" dependencies = [ "arrow", "arrow-array", @@ -5353,7 +5370,6 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-dynamodb", - "blake3", "byteorder", "bytes", "chrono", @@ -5383,8 +5399,9 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0847352699e869a465b49bd49ecbc467bc5a3dcb168757bf39368362aad05f99" dependencies = [ "arrow-array", "arrow-schema", @@ -5397,13 +5414,14 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.3#f7d475539cefbd140cc46a828f3d843e68cd10f1" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "254c1e9287789c8d744f93f2010a8389a5ea3fcedc0dddf7649c4117ec830c57" dependencies = [ - "frostem", "icu_segmenter", "jieba-rs", "lindera", + "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -5681,7 +5699,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum", + "strum 0.28.0", "strum_macros 0.28.0", "unicode-blocks", "unicode-normalization", @@ -5711,7 +5729,7 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum", + "strum 0.28.0", "strum_macros 0.28.0", "thiserror 2.0.18", ] @@ -5783,15 +5801,6 @@ dependencies = [ "hashbrown 0.16.1", ] -[[package]] -name = "lru" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" -dependencies = [ - "hashbrown 0.17.1", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -6429,9 +6438,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.58.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" +checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" dependencies = [ "async-trait", "bytes", @@ -6492,13 +6501,12 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" +checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" dependencies = [ "ctor 1.0.12", "opendal-core", - "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -6515,22 +6523,24 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" +checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" dependencies = [ "anyhow", - "base64 0.23.1", + "base64 0.22.1", "bytes", "futures", "http 1.5.0", + "http-body 1.1.0", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-core", + "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -6539,25 +6549,11 @@ dependencies = [ "web-time", ] -[[package]] -name = "opendal-http-transport-reqwest" -version = "0.58.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" -dependencies = [ - "bytes", - "futures", - "http 1.5.0", - "http-body 1.1.0", - "opendal-core", - "reqwest 0.13.4", -] - [[package]] name = "opendal-layer-concurrent-limit" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" +checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" dependencies = [ "futures", "http 1.5.0", @@ -6567,9 +6563,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" +checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" dependencies = [ "log", "opendal-core", @@ -6577,9 +6573,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" +checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" dependencies = [ "backon", "log", @@ -6588,9 +6584,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" +checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" dependencies = [ "opendal-core", "tokio", @@ -6598,17 +6594,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" +checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" dependencies = [ - "base64 0.23.1", + "base64 0.22.1", "bytes", "http 1.5.0", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6619,18 +6615,17 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" +checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" dependencies = [ - "base64 0.23.1", + "base64 0.22.1", "bytes", "http 1.5.0", "log", - "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6640,9 +6635,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" +checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" dependencies = [ "http 1.5.0", "opendal-core", @@ -6650,15 +6645,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" +checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" dependencies = [ "bytes", "http 1.5.0", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -6667,9 +6662,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" +checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" dependencies = [ "async-trait", "bytes", @@ -6677,7 +6672,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -6688,9 +6683,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" +checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" dependencies = [ "bytes", "goosefs-sdk", @@ -6702,9 +6697,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" +checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" dependencies = [ "bytes", "hf-xet", @@ -6712,21 +6707,22 @@ dependencies = [ "log", "opendal-core", "percent-encoding", + "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" +checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" dependencies = [ "bytes", "http 1.5.0", "log", "opendal-core", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -6735,18 +6731,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.58.1" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" +checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" dependencies = [ - "base64 0.23.1", + "base64 0.22.1", "bytes", - "crc-fast", + "crc32c", "http 1.5.0", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.41.0", + "quick-xml 0.39.4", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -8679,6 +8675,16 @@ dependencies = [ "ordered-multimap", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -9552,6 +9558,15 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + [[package]] name = "strum" version = "0.28.0" @@ -9574,6 +9589,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + [[package]] name = "strum_macros" version = "0.28.0" diff --git a/Cargo.toml b/Cargo.toml index 936660d78..a47e59ea7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.3", default-features = false, "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.3", "tag" = "v11.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=10.0.0", default-features = false } +lance-core = "=10.0.0" +lance-datagen = "=10.0.0" +lance-file = "=10.0.0" +lance-io = { "version" = "=10.0.0", default-features = false } +lance-index = "=10.0.0" +lance-linalg = "=10.0.0" +lance-namespace = "=10.0.0" +lance-namespace-impls = { "version" = "=10.0.0", default-features = false } +lance-table = "=10.0.0" +lance-testing = "=10.0.0" +lance-datafusion = "=10.0.0" +lance-encoding = "=10.0.0" +lance-arrow = "=10.0.0" ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index 3fa3b08db..11fca32d0 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -431,10 +431,9 @@ Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on th Resolves to `undefined` when the MemWAL LSM write path is not enabled (no spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)). -The returned spec mirrors what was passed to -[Table#setLsmWriteSpec](Table.md#setlsmwritespec), except that `maintainedIndexes` always -reports the concrete list resolved when the spec was set — `undefined` -never round-trips. +The returned spec — including its `maintainedIndexes` and +`writerConfigDefaults` — mirrors what was passed to +[Table#setLsmWriteSpec](Table.md#setlsmwritespec). #### Returns @@ -807,11 +806,6 @@ All variants require the table to have an unenforced primary key ([Table#setUnenforcedPrimaryKey](Table.md#setunenforcedprimarykey)); bucket sharding additionally requires it to be the single column being bucketed. -Omitting `maintainedIndexes` maintains every index on the table, resolved -here, failing if one cannot be maintained — name them to install anyway. -Naming them pins an exact set, and a still-building index is rejected -rather than quietly omitted. - #### Parameters * **spec**: [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md) diff --git a/docs/src/js/interfaces/LsmWriteSpec.md b/docs/src/js/interfaces/LsmWriteSpec.md index f2ae91186..8a588df6a 100644 --- a/docs/src/js/interfaces/LsmWriteSpec.md +++ b/docs/src/js/interfaces/LsmWriteSpec.md @@ -34,9 +34,7 @@ Bucket and identity variants: the sharding column. optional maintainedIndexes: string[]; ``` -Indexes the MemWAL keeps up to date. Omit to maintain every supported -index, resolved on install — a snapshot, so indexes created later are not -maintained. Pass `[]` for none. +Names of indexes the MemWAL should keep up to date during writes. *** diff --git a/java/pom.xml b/java/pom.xml index f85b5c906..153970f77 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.3 + 10.0.0 false 2.30.0 1.7 diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 04705475b..3359a2643 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -197,11 +197,7 @@ export interface LsmWriteSpec { column?: string; /** Bucket variant: the number of buckets, in `[1, 1024]`. */ numBuckets?: number; - /** - * Indexes the MemWAL keeps up to date. Omit to maintain every supported - * index, resolved on install — a snapshot, so indexes created later are not - * maintained. Pass `[]` for none. - */ + /** Names of indexes the MemWAL should keep up to date during writes. */ maintainedIndexes?: string[]; /** Default `ShardWriter` configuration recorded in the MemWAL index. */ writerConfigDefaults?: Record; @@ -599,11 +595,6 @@ export abstract class Table { * All variants require the table to have an unenforced primary key * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally * requires it to be the single column being bucketed. - * - * Omitting `maintainedIndexes` maintains every index on the table, resolved - * here, failing if one cannot be maintained — name them to install anyway. - * Naming them pins an exact set, and a still-building index is rejected - * rather than quietly omitted. * @param {LsmWriteSpec} spec The sharding spec to install. * @returns {Promise} * @example @@ -631,10 +622,9 @@ export abstract class Table { * * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}). - * The returned spec mirrors what was passed to - * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always - * reports the concrete list resolved when the spec was set — `undefined` - * never round-trips. + * The returned spec — including its `maintainedIndexes` and + * `writerConfigDefaults` — mirrors what was passed to + * {@link Table#setLsmWriteSpec}. * @returns {Promise} */ abstract getLsmWriteSpec(): Promise; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index c4ece20e2..a2df58223 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -772,8 +772,7 @@ pub struct LsmWriteSpec { pub column: Option, /// Bucket variant: the number of buckets, in `[1, 1024]`. pub num_buckets: Option, - /// Indexes the MemWAL keeps up to date. Omitted resolves every - /// maintainable index on install; an empty array means none. + /// Names of indexes the MemWAL should keep up to date during writes. pub maintained_indexes: Option>, /// Default `ShardWriter` configuration recorded in the MemWAL index. pub writer_config_defaults: Option>, @@ -783,6 +782,7 @@ impl TryFrom for lancedb::table::LsmWriteSpec { type Error = napi::Error; fn try_from(value: LsmWriteSpec) -> napi::Result { + let maintained = value.maintained_indexes.unwrap_or_default(); let writer_config_defaults = value.writer_config_defaults.unwrap_or_default(); let spec = match value.spec_type.as_str() { "bucket" => { @@ -809,7 +809,7 @@ impl TryFrom for lancedb::table::LsmWriteSpec { } }; Ok(spec - .with_maintained_indexes(value.maintained_indexes) + .with_maintained_indexes(maintained) .with_writer_config_defaults(writer_config_defaults)) } } @@ -827,7 +827,7 @@ impl From for LsmWriteSpec { spec_type: "bucket".to_string(), column: Some(column), num_buckets: Some(num_buckets), - maintained_indexes, + maintained_indexes: Some(maintained_indexes), writer_config_defaults: Some(writer_config_defaults), }, Native::Identity { @@ -838,7 +838,7 @@ impl From for LsmWriteSpec { spec_type: "identity".to_string(), column: Some(column), num_buckets: None, - maintained_indexes, + maintained_indexes: Some(maintained_indexes), writer_config_defaults: Some(writer_config_defaults), }, Native::Unsharded { @@ -848,7 +848,7 @@ impl From for LsmWriteSpec { spec_type: "unsharded".to_string(), column: None, num_buckets: None, - maintained_indexes, + maintained_indexes: Some(maintained_indexes), writer_config_defaults: Some(writer_config_defaults), }, } diff --git a/python/pyproject.toml b/python/pyproject.toml index ce71484de..47510b7ea 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -63,7 +63,7 @@ tests = [ "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", - "pylance==9.0.0rc1", + "pylance==10.0.0", "requests>=2.31.0", "datafusion>=54,<55", "opentelemetry-sdk>=1.30.0", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index f87fd3d13..fad2744d3 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -653,10 +653,9 @@ class LsmWriteSpec: def identity(column: str) -> "LsmWriteSpec": ... @staticmethod def unsharded() -> "LsmWriteSpec": ... - def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec": - """Set which indexes the MemWAL keeps up to date. None resolves every - index on the table at install, failing if one cannot be maintained; - a list is verbatim, empty means none.""" + def with_maintained_indexes(self, indexes: List[str]) -> "LsmWriteSpec": + """Return a copy of this spec asking the MemWAL to keep the named + indexes up to date as rows are appended.""" ... def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec": """Return a copy of this spec recording the given default @@ -671,9 +670,7 @@ class LsmWriteSpec: @property def num_buckets(self) -> Optional[int]: ... @property - def maintained_indexes(self) -> Optional[List[str]]: - """Indexes the MemWAL keeps up to date, or None for every supported one.""" - ... + def maintained_indexes(self) -> List[str]: ... @property def writer_config_defaults(self) -> Dict[str, str]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index c566fc532..328718cf9 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4676,13 +4676,6 @@ class AsyncTable: via [`set_unenforced_primary_key`]; bucket sharding additionally requires it to be the single column being bucketed. - By default the MemWAL maintains every index on the table, resolved - here — a snapshot, so an index created afterwards needs the spec unset - and set again. This fails if one cannot be maintained; name the set - with ``with_maintained_indexes`` to install anyway. That pins an exact - set (a still-building index is rejected, not omitted); ``[]`` maintains - none. - Parameters ---------- spec : LsmWriteSpec @@ -4709,9 +4702,9 @@ class AsyncTable: Returns ``None`` when the MemWAL LSM write path is not enabled (no spec has been set, or it was removed with `unset_lsm_write_spec`). - The returned spec mirrors what was passed to `set_lsm_write_spec`, - except that ``maintained_indexes`` always reports the concrete list - resolved when the spec was set — ``None`` never round-trips. + The returned spec — including its ``maintained_indexes`` and + ``writer_config_defaults`` — mirrors what was passed to + `set_lsm_write_spec`. """ return await self._inner.get_lsm_write_spec() diff --git a/python/python/tests/test_lsm_write_spec.py b/python/python/tests/test_lsm_write_spec.py index 218793b89..d38918f09 100644 --- a/python/python/tests/test_lsm_write_spec.py +++ b/python/python/tests/test_lsm_write_spec.py @@ -83,9 +83,7 @@ def test_lsm_write_spec_repr(): assert s.spec_type == "bucket" assert s.column == "id" assert s.num_buckets == 4 - # A fresh spec defers its maintained set to install time. - assert s.maintained_indexes is None - assert s.with_maintained_indexes([]).maintained_indexes == [] + assert s.maintained_indexes == [] assert "bucket" in repr(s) assert "id" in repr(s) assert "4" in repr(s) @@ -171,23 +169,18 @@ def test_get_lsm_write_spec(tmp_path): table.unset_lsm_write_spec() assert table.get_lsm_write_spec() is None - # Identity round-trips (column recovered from the schema). Leaving the - # maintained set to be inferred picks up the index on the table, so the - # spec reads back naming it rather than as "infer". + # Identity round-trips (column recovered from the schema). table.set_lsm_write_spec(LsmWriteSpec.identity("id")) spec = table.get_lsm_write_spec() assert spec.spec_type == "identity" assert spec.column == "id" - assert spec.maintained_indexes == [idx_name] table.unset_lsm_write_spec() - # Unsharded round-trips (no routing column). Opting out is distinct from - # the inferred default. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + # Unsharded round-trips (no routing column). + table.set_lsm_write_spec(LsmWriteSpec.unsharded()) spec = table.get_lsm_write_spec() assert spec.spec_type == "unsharded" assert spec.column is None - assert spec.maintained_indexes == [] @pytest.mark.asyncio diff --git a/python/python/tests/test_merge_insert_lsm.py b/python/python/tests/test_merge_insert_lsm.py index e74c21589..5674a05ab 100644 --- a/python/python/tests/test_merge_insert_lsm.py +++ b/python/python/tests/test_merge_insert_lsm.py @@ -544,7 +544,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path): table.create_index("text", config=FTS()) # No maintained indexes: the active memtable FTS arm cannot serve un-compacted # docs, so the search would silently omit them — reject instead. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + table.set_lsm_write_spec(LsmWriteSpec.unsharded()) with pytest.raises(Exception, match="maintained"): table.search("fox", query_type="fts", fts_columns="text").to_arrow() @@ -631,7 +631,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path): ) # Spec with NO maintained indexes: the base vector index's catch-up is untracked, # so the scanner rejects rather than risk dropping compacted-but-unindexed rows. - table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([])) + table.set_lsm_write_spec(LsmWriteSpec.unsharded()) with pytest.raises(Exception, match="maintained"): table.search([1.0] * VECTOR_DIM).to_arrow() diff --git a/python/src/table.rs b/python/src/table.rs index 20a93556f..119388708 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -246,22 +246,12 @@ impl From for MergeResult { } } -/// Render for `__repr__`, so the default reads as Python's `None` rather than -/// Rust's `Some([..])`. -fn fmt_maintained(maintained: &Option>) -> String { - match maintained { - Some(names) => format!("{:?}", names), - None => "None".to_string(), - } -} - /// Specification selecting Lance's MemWAL LSM-style write path for /// `merge_insert`. /// /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()` /// classmethods, then optionally chain `with_maintained_indexes(...)` and -/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the -/// MemWAL supports, resolved on install. +/// `with_writer_config_defaults(...)`. #[pyclass(from_py_object)] #[derive(Clone, Debug)] pub struct LsmWriteSpec { @@ -301,11 +291,11 @@ impl LsmWriteSpec { } } - /// Set which indexes the MemWAL maintains. `None` (the default) - /// resolves every supported index on install; a list is verbatim, - /// and an empty list maintains nothing. - #[pyo3(signature = (indexes))] - pub fn with_maintained_indexes(&self, indexes: Option>) -> Self { + /// Replace the list of indexes the MemWAL should keep up to date as + /// rows are appended. Each name must reference an index that + /// already exists on the table at the time `set_lsm_write_spec` + /// is called. + pub fn with_maintained_indexes(&self, indexes: Vec) -> Self { Self { inner: self.inner.clone().with_maintained_indexes(indexes), } @@ -327,29 +317,23 @@ impl LsmWriteSpec { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})", - column, - num_buckets, - fmt_maintained(maintained_indexes), - writer_config_defaults, + "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={:?}, writer_config_defaults={:?})", + column, num_buckets, maintained_indexes, writer_config_defaults, ), lancedb::table::LsmWriteSpec::Identity { column, maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})", - column, - fmt_maintained(maintained_indexes), - writer_config_defaults, + "LsmWriteSpec.identity(column={:?}, maintained_indexes={:?}, writer_config_defaults={:?})", + column, maintained_indexes, writer_config_defaults, ), lancedb::table::LsmWriteSpec::Unsharded { maintained_indexes, writer_config_defaults, } => format!( - "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})", - fmt_maintained(maintained_indexes), - writer_config_defaults, + "LsmWriteSpec.unsharded(maintained_indexes={:?}, writer_config_defaults={:?})", + maintained_indexes, writer_config_defaults, ), } } @@ -384,10 +368,10 @@ impl LsmWriteSpec { } } - /// Indexes the MemWAL keeps up to date, or `None` for every supported one. + /// Names of indexes the MemWAL should keep up to date during writes. #[getter] - pub fn maintained_indexes(&self) -> Option> { - self.inner.maintained_indexes().map(<[String]>::to_vec) + pub fn maintained_indexes(&self) -> Vec { + self.inner.maintained_indexes().to_vec() } /// Default `ShardWriter` configuration recorded by this spec. diff --git a/python/uv.lock b/python/uv.lock index 2cdcb182e..8af4d634f 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -799,7 +799,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -834,37 +834,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cublas" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime" }, ] cufft = [ - { name = "nvidia-cufft", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cufft" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti" }, ] curand = [ - { name = "nvidia-curand", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-curand" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cusolver" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cusparse" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" }, + { name = "nvidia-nvtx" }, ] [[package]] @@ -1023,7 +1023,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1440,16 +1440,16 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "cachetools", marker = "python_full_version < '3.11'" }, - { name = "certifi", marker = "python_full_version < '3.11'" }, - { name = "httpx", marker = "python_full_version < '3.11'" }, - { name = "ibm-cos-sdk", marker = "python_full_version < '3.11'" }, - { name = "lomond", marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "requests", marker = "python_full_version < '3.11'" }, - { name = "tabulate", marker = "python_full_version < '3.11'" }, - { name = "urllib3", marker = "python_full_version < '3.11'" }, + { name = "cachetools" }, + { name = "certifi" }, + { name = "httpx" }, + { name = "ibm-cos-sdk" }, + { name = "lomond" }, + { name = "packaging" }, + { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" } }, + { name = "requests" }, + { name = "tabulate" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c7/56/2e3df38a1f13062095d7bde23c87a92f3898982993a15186b1bfecbd206f/ibm_watsonx_ai-1.3.42.tar.gz", hash = "sha256:ee5be59009004245d957ce97d1227355516df95a2640189749487614fef674ff", size = 688651, upload-time = "2025-10-01T13:35:41.527Z" } wheels = [ @@ -1468,17 +1468,17 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "cachetools", marker = "python_full_version >= '3.11'" }, - { name = "certifi", marker = "python_full_version >= '3.11'" }, - { name = "httpx", marker = "python_full_version >= '3.11'" }, - { name = "ibm-cos-sdk", marker = "python_full_version >= '3.11'" }, - { name = "lomond", marker = "python_full_version >= '3.11'" }, - { name = "packaging", marker = "python_full_version >= '3.11'" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, + { name = "cachetools" }, + { name = "certifi" }, + { name = "httpx" }, + { name = "ibm-cos-sdk" }, + { name = "lomond" }, + { name = "packaging" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, - { name = "requests", marker = "python_full_version >= '3.11'" }, - { name = "tabulate", marker = "python_full_version >= '3.11'" }, - { name = "urllib3", marker = "python_full_version >= '3.11'" }, + { name = "requests" }, + { name = "tabulate" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/a3/c756b534696ab2f3f29882fdb7ca7198b7a5c94e10c0a3a327853d6d6b79/ibm_watsonx_ai-1.5.14.tar.gz", hash = "sha256:a756488bd57e87c0fc51be42dcba871143cfe0ac1e805c497c5047e1e4f13e9d", size = 735804, upload-time = "2026-06-22T12:32:43.85Z" } wheels = [ @@ -1554,17 +1554,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1583,18 +1583,18 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -1606,7 +1606,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -2005,7 +2005,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=1.10" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, - { name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" }, + { name = "pylance", marker = "extra == 'tests'", specifier = "==10.0.0" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" }, @@ -2858,7 +2858,7 @@ name = "nvidia-cudnn-cu13" version = "9.19.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, @@ -2870,7 +2870,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -2900,9 +2900,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -2914,7 +2914,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -3091,10 +3091,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } wheels = [ @@ -3143,11 +3143,11 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "pytz", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, - { name = "tzdata", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -3210,9 +3210,9 @@ resolution-markers = [ "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.14'" }, - { name = "tzdata", marker = "(python_full_version >= '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.14' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -3320,7 +3320,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -3912,7 +3912,7 @@ crypto = [ [[package]] name = "pylance" -version = "7.0.0" +version = "10.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace" }, @@ -3922,12 +3922,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/ad/2f64921bf346e7075aef24a72595db44821724a3d89a9a92dd24e79632aa/pylance-7.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:98422021975be76e72b1572f41b8c9abb3bee5bdc9bfa5e9ce731110a65ed4d1", size = 62134146, upload-time = "2026-05-27T21:59:37.459Z" }, - { url = "https://files.pythonhosted.org/packages/73/1c/c5a01bee0160b55d9a98895cbd33091d038f0a0995b121ab72e629008d02/pylance-7.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bec86ee5b6fbd8bfc493e653f0a1fba0303cfe5492b9b46fc25ab908edc7183", size = 65373684, upload-time = "2026-05-27T22:04:01.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/da/1fe8b8f7dbfe734d76af76acc994fc360a0d0c79a4874ef69f5a72a58fe3/pylance-7.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881491432c53184e52f8d1db8d5f872f39a03f36fb104bec77b33d379519d8b5", size = 69458555, upload-time = "2026-05-27T22:16:50.567Z" }, - { url = "https://files.pythonhosted.org/packages/76/f0/dd505cf3fd0226ab9d94759acd713125af1d3bfacfd80bbd52e3b9f89509/pylance-7.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18453999e7fff4f76b16d6b7882c9df0628bd142ff95e2461bd7dd5ee3fe0af3", size = 65394430, upload-time = "2026-05-27T22:05:30.923Z" }, - { url = "https://files.pythonhosted.org/packages/17/ba/2357b81034f28eb00790e258ed140289a6a887a7468ca9df6349fd186b27/pylance-7.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:04a58051d408c60fe76d41a220dcaf8fea8fb6d1aa0ca78a709b60bc3cc8d19a", size = 69473470, upload-time = "2026-05-27T22:17:18.935Z" }, - { url = "https://files.pythonhosted.org/packages/1f/ec/5c00b6303a67d787f9475141832cbdc513d674ac3dcaeef8a7b169905e65/pylance-7.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:467d4864af047eaab4e1370e2f1e88e2c6f507c079874421116cb41d78bc3629", size = 74792863, upload-time = "2026-05-27T22:19:23.875Z" }, + { url = "https://files.pythonhosted.org/packages/82/b2/c81de196076c4c8d768f485324a0043e113ce0950a339977d83fee9783ef/pylance-10.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d4bba56ae829202b7e9cdc82c92c00a4b52f03e679dc3edfad1db09d1285be2e", size = 69279797, upload-time = "2026-08-07T18:25:24.812Z" }, + { url = "https://files.pythonhosted.org/packages/d2/e9/af671a6225740bd70c1e70bd83fa08091d628b960397cbecc477f16aaaf1/pylance-10.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:489b944827c0271e16a62b4006f8c75acc3bd7fbc381f35388a2afaaae38d438", size = 72775384, upload-time = "2026-08-07T18:31:29.686Z" }, + { url = "https://files.pythonhosted.org/packages/5d/22/e07194195bb3bbdf062b0c31690fc92fccb2686d5e768efa1dc379a93350/pylance-10.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:018efe7d437d326b9049c1223458bd955e85e48509b4b0bbf8b2d3d94075dbff", size = 76632194, upload-time = "2026-08-07T18:44:29.001Z" }, + { url = "https://files.pythonhosted.org/packages/f2/71/ed9956cf657e86a5fee5d5ddcfa7bfed0d7c2968bee39ed85d12629d8c93/pylance-10.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:254bad5d765c14db6c4eddd5323133dee3c76ff6d40c4777afe0b0e91d4c04d2", size = 72804222, upload-time = "2026-08-07T18:31:24.264Z" }, + { url = "https://files.pythonhosted.org/packages/94/96/de449c246b2892df9d5a0af775b79a061e9897e6942bcee5c4ab6d6071f8/pylance-10.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9f0c089e30389d9b7765a4c16f3d9325b05ed4fbf6cc2bbc9418292983472e54", size = 76602015, upload-time = "2026-08-07T18:46:27.724Z" }, + { url = "https://files.pythonhosted.org/packages/53/81/4a5a9072b6d68c4dbb8c7b5530a381c7a6bea4ba92f9eca58ac885142722/pylance-10.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:9fecf46e6835dc64b2d71767d5eab4f73787543a852432e9ba8ecf8527b3cdab", size = 82799247, upload-time = "2026-08-07T18:47:46.855Z" }, ] [[package]] @@ -4683,10 +4683,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -4734,13 +4734,13 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "narwhals", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -4784,7 +4784,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -4843,7 +4843,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -4920,7 +4920,7 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ diff --git a/rust/lancedb/src/dataloader/permutation/shuffle.rs b/rust/lancedb/src/dataloader/permutation/shuffle.rs index 2bc06452c..3aa366af0 100644 --- a/rust/lancedb/src/dataloader/permutation/shuffle.rs +++ b/rust/lancedb/src/dataloader/permutation/shuffle.rs @@ -12,8 +12,7 @@ use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::{ reader::{FileReader, FileReaderOptions}, version::ConcreteFileVersion, - versions, - writer::FileWriterOptions, + writer::{FileWriter, FileWriterOptions}, }; use lance_io::{ ReadBatchParams, @@ -154,11 +153,13 @@ impl Shuffler { source: None, })?; let object_writer = object_store.create(&path).await?; - let writer = versions::create_writer( - ConcreteFileVersion::V2_1, + let writer = FileWriter::try_new( object_writer, schema.clone(), - FileWriterOptions::default(), + FileWriterOptions { + format_version: Some(ConcreteFileVersion::V2_1.into()), + ..Default::default() + }, )?; file_writers.push(writer); } diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 0d843dd54..c97085b74 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2520,9 +2520,9 @@ impl BaseTable for RemoteTable { self.check_mutable().await?; // Map the spec onto the server's request DTO. `sharding` is internally - // tagged on `mode` to mirror sophon's `Sharding` enum. A null - // `maintained_indexes` asks the server to resolve every maintainable - // index at HEAD; a list is verbatim, an empty one meaning none. + // tagged on `mode` to mirror sophon's `Sharding` enum; `maintained_indexes` + // and `writer_config_defaults` are sent verbatim (an empty list means "no + // maintained indexes", not "default to all"). let sharding = match &spec { LsmWriteSpec::Bucket { column, @@ -5955,18 +5955,17 @@ mod tests { .await .unwrap(); - // Positions are relative to the first retained token, so dropping the - // leading "hello" stop word does not shift the remaining tokens. + // Lance 10 retains original token positions after stop-word removal. assert_eq!( tokens, vec![ FtsToken { text: "こんにちは".to_string(), - position: 0, + position: 1, }, FtsToken { text: "世界".to_string(), - position: 1, + position: 2, }, ] ); @@ -6599,7 +6598,7 @@ mod tests { .unwrap() }); let spec = crate::table::LsmWriteSpec::unsharded() - .with_maintained_indexes(vec!["id_idx".to_string()]) + .with_maintained_indexes(["id_idx"]) .with_writer_config_defaults([("max_memtable_rows", "1000")]); table.set_lsm_write_spec(spec).await.unwrap(); } @@ -6618,29 +6617,11 @@ mod tests { body["sharding"], serde_json::json!({ "mode": "bucket", "column": "id", "num_buckets": 16 }) ); - // An unpinned maintained set sends null: resolve server-side. - assert_eq!(body["maintained_indexes"], serde_json::Value::Null); - http::Response::builder().status(200).body("{}").unwrap() - }); - table - .set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16)) - .await - .unwrap(); - } - - /// `[]` (none) must stay distinguishable on the wire from null (all). - #[tokio::test] - async fn test_set_lsm_write_spec_no_maintained_indexes() { - let table = Table::new_with_handler("my_table", |request| { - let body = request.body().unwrap().as_bytes().unwrap(); - let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!(body["maintained_indexes"], serde_json::json!([])); http::Response::builder().status(200).body("{}").unwrap() }); table - .set_lsm_write_spec( - crate::table::LsmWriteSpec::bucket("id", 16).with_maintained_indexes(Vec::new()), - ) + .set_lsm_write_spec(crate::table::LsmWriteSpec::bucket("id", 16)) .await .unwrap(); } @@ -6719,7 +6700,7 @@ mod tests { } => { assert_eq!(column, "id"); assert_eq!(num_buckets, 4); - assert_eq!(maintained_indexes, Some(vec!["id_idx".to_string()])); + assert_eq!(maintained_indexes, vec!["id_idx".to_string()]); assert_eq!( writer_config_defaults .get("durable_write") diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 5120c48b7..f64fa08b4 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -370,8 +370,6 @@ pub use self::merge::MergeResult; /// date) and [`LsmWriteSpec::with_writer_config_defaults`] (default /// `ShardWriter` configuration recorded in the MemWAL index). /// -/// A fresh spec maintains every index on the table, resolved on install. -/// /// Install a spec with [`Table::set_lsm_write_spec`] and remove it with /// [`Table::unset_lsm_write_spec`]. The actual `merge_insert` dispatch /// onto the MemWAL writer is a follow-up. @@ -386,12 +384,9 @@ pub enum LsmWriteSpec { Bucket { column: String, num_buckets: u32, - /// Indexes the MemWAL maintains in-memory as rows are appended. - /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. - maintained_indexes: Option>, + /// Names of indexes (already created on the table) that the + /// MemWAL should maintain in-memory as rows are appended. + maintained_indexes: Vec, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, @@ -401,41 +396,35 @@ pub enum LsmWriteSpec { /// distinct value of `column` becomes its own shard. Identity { column: String, - /// Indexes the MemWAL maintains in-memory as rows are appended. - /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. - maintained_indexes: Option>, + /// Names of indexes (already created on the table) that the + /// MemWAL should maintain in-memory as rows are appended. + maintained_indexes: Vec, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, /// No sharding — every `merge_insert` call writes to a single MemWAL shard. Unsharded { - /// Indexes the MemWAL maintains in-memory as rows are appended. - /// - /// `None` means every index it can maintain, resolved on install — a - /// snapshot, so indexes created later need the spec unset and re-set. - /// `Some([])` maintains nothing. - maintained_indexes: Option>, + /// Names of indexes (already created on the table) that the + /// MemWAL should maintain in-memory as rows are appended. + maintained_indexes: Vec, /// Default `ShardWriter` configuration recorded in the MemWAL index. writer_config_defaults: HashMap, }, } impl LsmWriteSpec { - /// Construct a hash-bucket sharding spec maintaining every index on the table. + /// Construct a hash-bucket sharding spec with no maintained indexes. pub fn bucket(column: impl Into, num_buckets: u32) -> Self { Self::Bucket { column: column.into(), num_buckets, - maintained_indexes: None, + maintained_indexes: Vec::new(), writer_config_defaults: HashMap::new(), } } /// Construct an identity-sharding spec (shard by the raw value of - /// `column`) maintaining every index on the table. + /// `column`) with no maintained indexes. /// /// `column` must be a deterministic function of the unenforced primary /// key: every row with a given primary key must always produce the same @@ -447,37 +436,28 @@ impl LsmWriteSpec { pub fn identity(column: impl Into) -> Self { Self::Identity { column: column.into(), - maintained_indexes: None, + maintained_indexes: Vec::new(), writer_config_defaults: HashMap::new(), } } - /// Construct an unsharded spec maintaining every index on the table. + /// Construct an unsharded spec with no maintained indexes. pub fn unsharded() -> Self { Self::Unsharded { - maintained_indexes: None, + maintained_indexes: Vec::new(), writer_config_defaults: HashMap::new(), } } - /// Set which indexes the MemWAL maintains. - /// - /// `None` (the default) resolves to every index on the table at install, - /// failing if one cannot be maintained — name the set to install anyway. A - /// list is verbatim: each name must already exist and be maintainable, and - /// an empty list maintains nothing. - /// - /// ``` - /// # use lancedb::table::LsmWriteSpec; - /// // Every index the table has when the spec is installed: - /// LsmWriteSpec::unsharded().with_maintained_indexes(None); - /// // Exactly these: - /// LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_idx".to_string()]); - /// // None at all: - /// LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new()); - /// ``` - pub fn with_maintained_indexes(mut self, indexes: impl Into>>) -> Self { - let indexes = indexes.into(); + /// Replace the list of indexes the MemWAL should keep up to date as + /// rows are appended. Each name must reference an index that already + /// exists on the table at the time `set_lsm_write_spec` is called. + pub fn with_maintained_indexes(mut self, indexes: I) -> Self + where + I: IntoIterator, + S: Into, + { + let v: Vec = indexes.into_iter().map(Into::into).collect(); match &mut self { Self::Bucket { maintained_indexes, .. @@ -487,7 +467,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => *maintained_indexes = indexes, + } => *maintained_indexes = v, } self } @@ -523,9 +503,8 @@ impl LsmWriteSpec { self } - /// Borrow the list of index names this spec asks MemWAL to maintain, or - /// `None` when it asks for every index on the table. - pub fn maintained_indexes(&self) -> Option<&[String]> { + /// Borrow the list of index names this spec asks MemWAL to maintain. + pub fn maintained_indexes(&self) -> &[String] { match self { Self::Bucket { maintained_indexes, .. @@ -535,7 +514,7 @@ impl LsmWriteSpec { } | Self::Unsharded { maintained_indexes, .. - } => maintained_indexes.as_deref(), + } => maintained_indexes, } } @@ -1733,7 +1712,7 @@ impl Table { /// # async fn example(table: &Table) -> Result<(), Box> { /// table /// .set_lsm_write_spec( - /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(vec!["id_idx".to_string()]), + /// LsmWriteSpec::bucket("id", 16).with_maintained_indexes(["id_idx"]), /// ) /// .await?; /// # Ok(()) @@ -1755,10 +1734,9 @@ impl Table { /// /// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no /// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]). - /// The returned spec mirrors what was passed to - /// [`Table::set_lsm_write_spec`], except that - /// [`LsmWriteSpec::maintained_indexes`] always reports the concrete list - /// resolved when the spec was set — `None` never round-trips. + /// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and + /// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to + /// [`Table::set_lsm_write_spec`]. /// /// # Example /// @@ -5107,7 +5085,7 @@ mod tests { // Bucket spec round-trips exactly, including the routing column (recovered // from its field id), maintained indexes, and writer config defaults. let spec = LsmWriteSpec::bucket("id", 4) - .with_maintained_indexes(vec![idx_name.clone()]) + .with_maintained_indexes([idx_name]) .with_writer_config_defaults([("durable_write", "false")]); table.set_lsm_write_spec(spec.clone()).await.unwrap(); assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); @@ -5117,125 +5095,15 @@ mod tests { assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); // Identity sharding round-trips (column recovered from the schema). - // A spec left at its default maintains every index on the table, so it - // reads back naming the one on the table rather than as "infer". let spec = LsmWriteSpec::identity("region"); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name.clone()])) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); table.unset_lsm_write_spec().await.unwrap(); // Unsharded round-trips (no routing column). let spec = LsmWriteSpec::unsharded(); table.set_lsm_write_spec(spec.clone()).await.unwrap(); - assert_eq!( - table.get_lsm_write_spec().await.unwrap(), - Some(spec.with_maintained_indexes(vec![idx_name])) - ); - } - - /// The maintained set defaults to every index on the table, resolved at - /// install. An index the memtable cannot build fails the install rather - /// than being dropped: maintaining it would take the table offline for - /// writes, dropping it would hide that from the caller. - #[tokio::test] - async fn test_set_lsm_write_spec_infers_maintained_indexes() { - let tmp_dir = tempdir().unwrap(); - let uri = tmp_dir.path().to_str().unwrap(); - - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("tag", DataType::Utf8, true), - ])); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["a", "b", "c"])), - ], - ) - .unwrap(); - let reader: Box = - Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())); - let conn = ConnectBuilder::new(uri) - .read_consistency_interval(Duration::from_secs(0)) - .execute() - .await - .unwrap(); - let table = conn.create_table("t", reader).execute().await.unwrap(); - - table - .create_index(&["id"], Index::BTree(Default::default())) - .name("id_btree".to_string()) - .execute() - .await - .unwrap(); - table - .create_index(&["tag"], Index::Bitmap(Default::default())) - .name("tag_bitmap".to_string()) - .execute() - .await - .unwrap(); - - // Explicitly naming the bitmap index fails before anything commits. - let err = table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec!["tag_bitmap".to_string()]), - ) - .await - .unwrap_err(); - assert!( - matches!(err, Error::InvalidInput { ref message } if message.contains("tag_bitmap")), - "expected the bitmap index to be rejected, got {err:?}" - ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - - // The default covers every index, so the bitmap fails it too. - let err = table - .set_lsm_write_spec(LsmWriteSpec::unsharded()) - .await - .unwrap_err(); - assert!( - matches!(err, Error::InvalidInput { ref message } - if message.contains("tag_bitmap") && message.contains("maintained_indexes")), - "expected the inferred set to be rejected, got {err:?}" - ); - assert_eq!(table.get_lsm_write_spec().await.unwrap(), None); - - // Naming the maintainable subset installs. - table - .set_lsm_write_spec( - LsmWriteSpec::unsharded().with_maintained_indexes(vec!["id_btree".to_string()]), - ) - .await - .unwrap(); - assert_eq!( - table - .get_lsm_write_spec() - .await - .unwrap() - .unwrap() - .maintained_indexes(), - Some(["id_btree".to_string()].as_slice()) - ); - - // Opting out entirely is distinct from the default. - table.unset_lsm_write_spec().await.unwrap(); - table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(Vec::new())) - .await - .unwrap(); - assert_eq!( - table - .get_lsm_write_spec() - .await - .unwrap() - .unwrap() - .maintained_indexes(), - Some([].as_slice()) - ); + assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec)); } #[tokio::test] @@ -5340,7 +5208,7 @@ mod tests { use lance::dataset::WriteDestination; use lance::dataset::transaction::{DataOverlayGroup, Operation}; use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; - use lance_file::writer::FileWriterOptions; + use lance_file::writer::{FileWriter, FileWriterOptions}; use lance_io::utils::CachedFileSize; use lance_table::format::DataFile; use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; @@ -5411,11 +5279,13 @@ mod tests { let store = dataset.object_store(None).await.unwrap(); let path = dataset.data_dir().child(filename.clone()); let obj_writer = store.create(&path).await.unwrap(); - let mut writer = lance_file::versions::create_writer( - file_version, + let mut writer = FileWriter::try_new( obj_writer, overlay_schema, - FileWriterOptions::default(), + FileWriterOptions { + format_version: Some(file_version.into()), + ..Default::default() + }, ) .unwrap(); writer diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 3a5b6882d..82a1d1473 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -1161,7 +1161,7 @@ mod lsm_tests { .unwrap(); let fts_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![fts_index])) + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([fts_index])) .await .unwrap(); @@ -1254,7 +1254,7 @@ mod lsm_tests { .unwrap(); let vec_index = table.list_indices().await.unwrap()[0].name.clone(); table - .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes(vec![vec_index])) + .set_lsm_write_spec(LsmWriteSpec::unsharded().with_maintained_indexes([vec_index])) .await .unwrap(); diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 87c427b3c..0eb7c0231 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -29,7 +29,6 @@ use arrow_schema::{DataType, Schema as ArrowSchema, SchemaRef}; use lance::Dataset; use lance::dataset::mem_wal::{ DatasetMemWalExt, ShardWriter, ShardWriterConfig, evaluate_sharding_spec, - validate_maintained_indexes, }; use lance::index::DatasetIndexExt; use lance_core::datatypes::Schema as LanceSchema; @@ -38,9 +37,8 @@ use tokio::sync::RwLock; use uuid::Uuid; use crate::error::{Error, Result}; -use crate::index::IndexConfig; use crate::table::merge::{MergeInsertBuilder, MergeResult}; -use crate::table::{BaseTable, LsmWriteSpec, NativeTable}; +use crate::table::{LsmWriteSpec, NativeTable}; /// Spec id of the sole sharding spec installed by [`set_lsm_write_spec`]. /// Must match Lance's `InitializeMemWalBuilder` (`SHARDING_SPEC_ID`). @@ -82,44 +80,32 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) } } - // Before the builder borrows the dataset clone. `list_indices` merges an - // index's segments into one entry, so the result needs no dedup. - let maintained_indexes = { - let dataset = table.dataset.get().await?; - resolve_maintained_indexes( - &dataset, - &table.list_indices().await?, - spec.maintained_indexes(), - ) - .await? - }; - let mut dataset = (*table.dataset.get().await?).clone(); let mut builder = dataset.initialize_mem_wal(); - let writer_config_defaults = match spec { + let (maintained_indexes, writer_config_defaults) = match spec { LsmWriteSpec::Bucket { column, num_buckets, + maintained_indexes, writer_config_defaults, - .. } => { builder = builder.bucket_sharding(column, num_buckets); - writer_config_defaults + (maintained_indexes, writer_config_defaults) } LsmWriteSpec::Identity { column, + maintained_indexes, writer_config_defaults, - .. } => { builder = builder.identity_sharding(column); - writer_config_defaults + (maintained_indexes, writer_config_defaults) } LsmWriteSpec::Unsharded { + maintained_indexes, writer_config_defaults, - .. } => { builder = builder.unsharded(); - writer_config_defaults + (maintained_indexes, writer_config_defaults) } }; builder = builder.maintained_indexes(maintained_indexes); @@ -131,58 +117,6 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) Ok(()) } -/// Resolve a spec's maintained-index selection against `indices`, as reported -/// by [`Table::list_indices`](crate::Table::list_indices). -/// -/// `None` means every index on the table, snapshotted now. Lance validates -/// either selection against its shard-writer rules, so a spec that installs is -/// one the MemWAL can open. -/// -/// An unmaintainable index fails an inferred set rather than being dropped from -/// it — dropping would leave the caller believing it is maintained. -async fn resolve_maintained_indexes( - dataset: &Dataset, - indices: &[IndexConfig], - requested: Option<&[String]>, -) -> Result> { - let Some(requested) = requested else { - let all: Vec = indices.iter().map(|index| index.name.clone()).collect(); - validate_maintained_indexes(dataset, &all) - .await - .map_err(|source| Error::InvalidInput { - message: format!( - "cannot maintain every index on this table: {source}. Set \ - maintained_indexes explicitly to choose from {}", - index_name_list(indices), - ), - })?; - return Ok(all); - }; - for name in requested { - if !indices.iter().any(|index| &index.name == name) { - return Err(Error::InvalidInput { - message: format!( - "maintained index '{}' does not exist on this table; it has {}", - name, - index_name_list(indices), - ), - }); - } - } - validate_maintained_indexes(dataset, requested).await?; - Ok(requested.to_vec()) -} - -/// Index names for an error message. -fn index_name_list(indices: &[IndexConfig]) -> String { - if indices.is_empty() { - return "no indexes".to_string(); - } - let mut names: Vec<&str> = indices.iter().map(|index| index.name.as_str()).collect(); - names.sort_unstable(); - format!("[{}]", names.join(", ")) -} - // ============================================================================= // unset_lsm_write_spec // =============================================================================