From 10625f6ddaef2a5241f23036210d2c3c8b4b1424 Mon Sep 17 00:00:00 2001 From: Yang Cen Date: Wed, 4 Nov 2026 19:35:36 +0800 Subject: [PATCH] chore: update lance dependency to v15.0.0-beta.9 (#4814) Update Rust Lance dependencies to [v15.0.0-beta.9](https://github.com/lance-format/lance/releases/tag/v15.0.0-beta.9), including the scoped stable-row-ID conversion fix. Python tests use published beta.9 wheels on Linux/Windows while macOS retains beta.4. Java stays on beta.8, matching the original base, so this update does not depend on beta.9 Maven publication or macOS Java packaging. The dependency updater preserves workspace inheritance, the WAL test wrapper uses the new StoredBatch module, and Linux/remote Rust CI uses its installed mold linker. The Python lockfile also records the already-declared OCI extra. CLI CI now caches Cargo downloads under a separate key instead of restoring compiled targets: the previous runner exhausted its disk during cache restoration, before compilation. Its test matrix and commands are unchanged. This branch includes #4820's row-freshness changes: successful rows become current while skipped rows retain their values and remain eligible for retry. Repair identities, replay handling and Restore fencing protect publication completion. BREAKING CHANGE: `StalenessPlan::row_dirty` is a new public field. Rust callers using exhaustive struct literals must initialize it. Validation: - All reported non-Java checks passed on `f5fa66fcbcf31a8eb188935f728735560c77c873`, including Rust lint/Linux/remote, Python, NodeJS and CLI checks. Current head `c660bc0ac9fd5920ad4b64be9c19f732ef4c56a5` preserves those product sources, manifests, lockfiles and compiler/test commands; only the Java pin restoration and CLI cache configuration differ. Current-head CI is tracked separately. - The complete Java directory is identical to the original PR base. The scope correction reuses [Xuanwo's prepared and validated commit](https://github.com/lancedb/lancedb/pull/4814#issuecomment-6626088466); no new Java build was run. - CLI workflow YAML parses successfully, its inputs match the pinned rust-cache action schema, and the existing jobs, matrix and commands are unchanged. Hosted execution must confirm the cache-restoration repair. - Existing validation includes the updater regressions and 32 CI-script tests, dependency-policy checks, count/WAL coverage, and #4820's 42 freshness tests, strict core Clippy and Rust doctest. Root Ruff previously reported unchanged documentation-example issues; Python-package Ruff passed. This PR adopts the existing released Lance beta.9. A subsequent official SDK release and normal Sophon dependency adoption remain separate delivery steps. --------- Co-authored-by: Xuanwo --- .github/workflows/cli.yml | 5 + .github/workflows/rust.yml | 4 + Cargo.lock | 98 +- Cargo.toml | 30 +- ci/set_lance_version.py | 3 + ci/tests/test_set_lance_version.py | 19 + python/pyproject.toml | 2 +- python/uv.lock | 506 +++++--- rust/lancedb/Cargo.toml | 2 +- rust/lancedb/src/table/freshness.rs | 1678 +++++++++++++++++++++++++-- rust/lancedb/src/table/merge.rs | 2 +- rust/lancedb/src/table/refresh.rs | 4 +- 12 files changed, 2053 insertions(+), 300 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 95c68403a..8d8763e1d 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -24,6 +24,11 @@ jobs: with: components: clippy - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 + with: + # Restoring compiled targets can exhaust the hosted runner disk. + prefix-key: v1-cli-downloads + cache-targets: false + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Linux prerequisites if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y protobuf-compiler libssl-dev clang diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 07ad76d7f..1c64a2cad 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -180,6 +180,8 @@ jobs: # Need up-to-date compilers for kernels CC: clang-18 CXX: clang++-18 + # Preserve the repository CPU flags and use setup-mold's system linker. + RUSTFLAGS: "-C target-cpu=haswell -C target-feature=+avx2,+fma,+f16c -C linker-features=-lld" steps: - uses: actions/checkout@v6 with: @@ -248,6 +250,8 @@ jobs: env: CC: clang-18 CXX: clang++-18 + # Rust's default rust-lld bypasses setup-mold's /usr/bin/ld replacement. + RUSTFLAGS: "-C target-cpu=haswell -C target-feature=+avx2,+fma,+f16c -C linker-features=-lld" GH_TOKEN: ${{ github.token }} SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }} steps: diff --git a/Cargo.lock b/Cargo.lock index 2dae1cbcf..5eb2c94ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3752,8 +3752,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5250,8 +5250,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arc-swap", "arrow", @@ -5326,8 +5326,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-buffer", @@ -5349,7 +5349,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "59.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-buffer", @@ -5363,7 +5363,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "59.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-schema", @@ -5372,8 +5372,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrayref", "crunchy", @@ -5383,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-buffer", @@ -5421,8 +5421,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow", "arrow-array", @@ -5454,8 +5454,8 @@ dependencies = [ [[package]] name = "lance-datafusion-exec" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow", "arrow-array", @@ -5480,8 +5480,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow", "arrow-array", @@ -5498,8 +5498,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "proc-macro2", "quote", @@ -5508,8 +5508,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-arith", "arrow-array", @@ -5543,8 +5543,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-arith", "arrow-array", @@ -5576,8 +5576,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "geo-traits", "geo-types", @@ -5589,8 +5589,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arc-swap", "arrow", @@ -5666,8 +5666,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-schema", @@ -5690,8 +5690,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow", "arrow-array", @@ -5736,8 +5736,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "byteorder", "ethnum", @@ -5754,8 +5754,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "argminmax 0.7.1", "arrow-array", @@ -5770,8 +5770,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow", "async-trait", @@ -5785,8 +5785,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arc-swap", "arrow", @@ -5840,8 +5840,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-buffer", @@ -5855,8 +5855,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow", "arrow-array", @@ -5897,8 +5897,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "arrow-array", "arrow-schema", @@ -5911,8 +5911,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "15.0.0-beta.8" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.8#bcdd78e302d329ed2b7124116c396c8b7e3c95f8" +version = "15.0.0-beta.9" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.9#025b3d93aded5a201508a50e20e031a031c63b66" dependencies = [ "frostem", "icu_segmenter", @@ -7294,7 +7294,6 @@ version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4" dependencies = [ - "ctor", "opendal-core", "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", @@ -9496,6 +9495,7 @@ checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" dependencies = [ "bytemuck", "byteorder", + "serde", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 07339add0..ac10aab52 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,21 +19,21 @@ categories = ["database-implementations"] rust-version = "1.95.0" [workspace.dependencies] -lance = { "version" = "=15.0.0-beta.8", default-features = false, "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=15.0.0-beta.8", default-features = false, "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-jsonb = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=15.0.0-beta.8", default-features = false, "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=15.0.0-beta.8", "tag" = "v15.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=15.0.0-beta.9", default-features = false, "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=15.0.0-beta.9", default-features = false, "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-jsonb = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=15.0.0-beta.9", default-features = false, "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=15.0.0-beta.9", "tag" = "v15.0.0-beta.9", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" anyhow = "1" diff --git a/ci/set_lance_version.py b/ci/set_lance_version.py index bef97f8ad..406483a86 100644 --- a/ci/set_lance_version.py +++ b/ci/set_lance_version.py @@ -155,6 +155,9 @@ def update_cargo_toml(line_updater): lines = core_manifest.read_text().splitlines(keepends=True) for index, line in enumerate(lines): if line.startswith("lance-arrow ="): + dependency = tomllib.loads(line)["lance-arrow"] + if isinstance(dependency, dict) and dependency.get("workspace"): + continue updated = tomllib.loads(line_updater(line))["lance-arrow"] config = {"version": updated} if isinstance(updated, str) else updated if "path" in config: diff --git a/ci/tests/test_set_lance_version.py b/ci/tests/test_set_lance_version.py index 1173eab84..1cc0a319c 100644 --- a/ci/tests/test_set_lance_version.py +++ b/ci/tests/test_set_lance_version.py @@ -146,6 +146,25 @@ lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], " f"{command!r} not found in {commands!r}", ) + with (workdir / "Cargo.toml").open("a") as manifest: + manifest.write('lance-arrow = "1.0.0"\n') + for inherited in ( + "lance-arrow = { workspace = true }", + "lance-arrow.workspace = true", + ): + inherited_manifest = f"[dependencies]\n{inherited}\n" + core_manifest.write_text(inherited_manifest) + result = subprocess.run( + [sys.executable, str(SCRIPT), version], + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(core_manifest.read_text(), inherited_manifest) + def _write_fake_executables(self, fake_bin: Path) -> None: cargo = fake_bin / "cargo" cargo.write_text( diff --git a/python/pyproject.toml b/python/pyproject.toml index 6294879f2..a8005f973 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -65,7 +65,7 @@ tests = [ "pyarrow<25", "pyarrow-stubs>=16.0", # pylance 15.0.0b5+ has not published macOS wheels. - "pylance==15.0.0b7; sys_platform != 'darwin'", + "pylance==15.0.0b9; sys_platform != 'darwin'", "pylance==15.0.0b4; sys_platform == 'darwin'", "requests>=2.31.0", "datafusion>=54,<55", diff --git a/python/uv.lock b/python/uv.lock index dee49fb07..2bc641cca 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1,16 +1,20 @@ version = 1 -revision = 5 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] @@ -249,6 +253,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321, upload-time = "2024-02-06T09:43:09.663Z" }, ] +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -660,6 +673,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/52/d5bee5b6ea81882d549b566d2545b044bbcbc33fe5fbe001008a7e745a21/charset_normalizer-3.4.8-py3-none-any.whl", hash = "sha256:b7c1fb310df524e01fbe84d43b7f95aa4f808f8eaa0dafc185f64ba395e37d54", size = 64279, upload-time = "2026-07-06T15:27:57.043Z" }, ] +[[package]] +name = "circuitbreaker" +version = "2.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/ac/de7a92c4ed39cba31fe5ad9203b76a25ca67c530797f6bb420fff5f65ccb/circuitbreaker-2.1.3.tar.gz", hash = "sha256:1a4baee510f7bea3c91b194dcce7c07805fe96c4423ed5594b75af438531d084", size = 10787, upload-time = "2025-03-31T08:12:08.963Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/34/15f08edd4628f65217de1fc3c1a27c82e46fe357d60c217fc9881e12ebcc/circuitbreaker-2.1.3-py3-none-any.whl", hash = "sha256:87ba6a3ed03fdc7032bc175561c2b04d52ade9d5faf94ca2b035fbdc5e6b1dd1", size = 7737, upload-time = "2025-03-31T08:12:07.802Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -741,6 +763,82 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] +[[package]] +name = "crc32c" +version = "2.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/66/7e97aa77af7cf6afbff26e3651b564fe41932599bc2d3dce0b2f73d4829a/crc32c-2.8.tar.gz", hash = "sha256:578728964e59c47c356aeeedee6220e021e124b9d3e8631d95d9a5e5f06e261c", size = 48179, upload-time = "2025-10-17T06:20:13.61Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/a0/28b4686a8db0bb0f77970f4c6ccede90d1d5740a1d4b4703bd54c3e75655/crc32c-2.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2c0f4eb01fe7c0a3e3f973a418e04d52101bb077dd77626fd80c658ec60aaf95", size = 66321, upload-time = "2025-10-17T06:18:53.543Z" }, + { url = "https://files.pythonhosted.org/packages/76/1f/1697f5b8b770f715ed9b264d79e36b4f77ae0527f81f3c749ef08937a32e/crc32c-2.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6baefcfbca82b1a9678455416da24f18629769a76920c640d5a538620a7d12bb", size = 62985, upload-time = "2025-10-17T06:18:54.97Z" }, + { url = "https://files.pythonhosted.org/packages/e0/e5/333cfa5ffa8d5779733aced2b984b5e5139b4a8ceaa2c6bc563e9a1092f3/crc32c-2.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7f959fcf6c5aad1c4a653ee1a50f05760dab1d1c35d98ec4d7f0f68643f7612", size = 61517, upload-time = "2025-10-17T06:18:55.795Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d8/362a009e8140dd926a153b44d56753e3aa7cb50aca243779a84adadbff11/crc32c-2.8-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bb678507a4e4cf3f0506607b046ecc4ed1c58a19e08a3fb3c2d25441c480bf1", size = 79385, upload-time = "2025-10-17T06:18:56.598Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0d4ea3aa71ffb15f1285669d23024cc40779388ce32157d339dc2584491c/crc32c-2.8-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1a16f7ffa4c242a909558565567cbba95148603717b53538ea299c98da68e7a9", size = 80965, upload-time = "2025-10-17T06:18:57.384Z" }, + { url = "https://files.pythonhosted.org/packages/20/44/d77657aaca4a2c0283f2356a3da6f8e91b003567bb8f09daaf540cbf192f/crc32c-2.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0184369aad562d801f91f454c81f56b9ecb966f6b96684c4d6cf82fc8741d2ad", size = 79993, upload-time = "2025-10-17T06:18:58.503Z" }, + { url = "https://files.pythonhosted.org/packages/ab/c0/07017a93ebf85d9408028b7e03ef96d5c6bfb14cb77cfe90d35eedcc1501/crc32c-2.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:86d2eeb5f0189bd803720abe7387019328ea34c4acde62999e5723f789bc316b", size = 79243, upload-time = "2025-10-17T06:18:59.273Z" }, + { url = "https://files.pythonhosted.org/packages/c7/1a/b3c5ac4cf2fd1f82395173d0bd8e1a15d09f0bc1eccdf10ea7f8caaccd67/crc32c-2.8-cp310-cp310-win32.whl", hash = "sha256:51da61904a9e753780a2e6011885677d601db1fa840be4b68799643a113e6f08", size = 64888, upload-time = "2025-10-17T06:19:00.089Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f2/60c45fc7bb2221d3c93c7a872e921be591f40d45228fe46f879b1d8c0424/crc32c-2.8-cp310-cp310-win_amd64.whl", hash = "sha256:b2d6a1f2500daaf2e4b08f97ad0349aa2eff5faaaa5fd3350314a26eade334cd", size = 66639, upload-time = "2025-10-17T06:19:00.974Z" }, + { url = "https://files.pythonhosted.org/packages/dc/0b/5e03b22d913698e9cc563f39b9f6bbd508606bf6b8e9122cd6bf196b87ea/crc32c-2.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e560a97fbb96c9897cb1d9b5076ef12fc12e2e25622530a1afd0de4240f17e1f", size = 66329, upload-time = "2025-10-17T06:19:01.771Z" }, + { url = "https://files.pythonhosted.org/packages/6b/38/2fe0051ffe8c6a650c8b1ac0da31b8802d1dbe5fa40a84e4b6b6f5583db5/crc32c-2.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6762d276d90331a490ef7e71ffee53b9c0eb053bd75a272d786f3b08d3fe3671", size = 62988, upload-time = "2025-10-17T06:19:02.953Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/5837a71c014be83aba1469c58820d287fc836512a0cad6b8fdd43868accd/crc32c-2.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:60670569f5ede91e39f48fb0cb4060e05b8d8704dd9e17ede930bf441b2f73ef", size = 61522, upload-time = "2025-10-17T06:19:03.796Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/63972fc1452778e2092ae998c50cbfc2fc93e3fa9798a0278650cd6169c5/crc32c-2.8-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:711743da6ccc70b3c6718c328947b0b6f34a1fe6a6c27cc6c1d69cc226bf70e9", size = 80200, upload-time = "2025-10-17T06:19:04.617Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3a/60eb49d7bdada4122b3ffd45b0df54bdc1b8dd092cda4b069a287bdfcff4/crc32c-2.8-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5eb4094a2054774f13b26f21bf56792bb44fa1fcee6c6ad099387a43ffbfb4fa", size = 81757, upload-time = "2025-10-17T06:19:05.496Z" }, + { url = "https://files.pythonhosted.org/packages/f5/63/6efc1b64429ef7d23bd58b75b7ac24d15df327e3ebbe9c247a0f7b1c2ed1/crc32c-2.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fff15bf2bd3e95780516baae935ed12be88deaa5ebe6143c53eb0d26a7bdc7b7", size = 80830, upload-time = "2025-10-17T06:19:06.621Z" }, + { url = "https://files.pythonhosted.org/packages/e1/eb/0ae9f436f8004f1c88f7429e659a7218a3879bd11a6b18ed1257aad7e98b/crc32c-2.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c0e11e3826668121fa53e0745635baf5e4f0ded437e8ff63ea56f38fc4f970a", size = 80095, upload-time = "2025-10-17T06:19:07.381Z" }, + { url = "https://files.pythonhosted.org/packages/9e/81/4afc9d468977a4cd94a2eb62908553345009a7c0d30e74463a15d4b48ec3/crc32c-2.8-cp311-cp311-win32.whl", hash = "sha256:38f915336715d1f1353ab07d7d786f8a789b119e273aea106ba55355dfc9101d", size = 64886, upload-time = "2025-10-17T06:19:08.497Z" }, + { url = "https://files.pythonhosted.org/packages/d6/e8/94e839c9f7e767bf8479046a207afd440a08f5c59b52586e1af5e64fa4a0/crc32c-2.8-cp311-cp311-win_amd64.whl", hash = "sha256:60e0a765b1caab8d31b2ea80840639253906a9351d4b861551c8c8625ea20f86", size = 66639, upload-time = "2025-10-17T06:19:09.338Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/fd18ef23c42926b79c7003e16cb0f79043b5b179c633521343d3b499e996/crc32c-2.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:572ffb1b78cce3d88e8d4143e154d31044a44be42cb3f6fbbf77f1e7a941c5ab", size = 66379, upload-time = "2025-10-17T06:19:10.115Z" }, + { url = "https://files.pythonhosted.org/packages/7f/b8/c584958e53f7798dd358f5bdb1bbfc97483134f053ee399d3eeb26cca075/crc32c-2.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cf827b3758ee0c4aacd21ceca0e2da83681f10295c38a10bfeb105f7d98f7a68", size = 63042, upload-time = "2025-10-17T06:19:10.946Z" }, + { url = "https://files.pythonhosted.org/packages/62/e6/6f2af0ec64a668a46c861e5bc778ea3ee42171fedfc5440f791f470fd783/crc32c-2.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:106fbd79013e06fa92bc3b51031694fcc1249811ed4364ef1554ee3dd2c7f5a2", size = 61528, upload-time = "2025-10-17T06:19:11.768Z" }, + { url = "https://files.pythonhosted.org/packages/17/8b/4a04bd80a024f1a23978f19ae99407783e06549e361ab56e9c08bba3c1d3/crc32c-2.8-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6dde035f91ffbfe23163e68605ee5a4bb8ceebd71ed54bb1fb1d0526cdd125a2", size = 80028, upload-time = "2025-10-17T06:19:12.554Z" }, + { url = "https://files.pythonhosted.org/packages/21/8f/01c7afdc76ac2007d0e6a98e7300b4470b170480f8188475b597d1f4b4c6/crc32c-2.8-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e41ebe7c2f0fdcd9f3a3fd206989a36b460b4d3f24816d53e5be6c7dba72c5e1", size = 81531, upload-time = "2025-10-17T06:19:13.406Z" }, + { url = "https://files.pythonhosted.org/packages/32/2b/8f78c5a8cc66486be5f51b6f038fc347c3ba748d3ea68be17a014283c331/crc32c-2.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ecf66cf90266d9c15cea597d5cc86c01917cd1a238dc3c51420c7886fa750d7e", size = 80608, upload-time = "2025-10-17T06:19:14.223Z" }, + { url = "https://files.pythonhosted.org/packages/db/86/fad1a94cdeeeb6b6e2323c87f970186e74bfd6fbfbc247bf5c88ad0873d5/crc32c-2.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:59eee5f3a69ad0793d5fa9cdc9b9d743b0cd50edf7fccc0a3988a821fef0208c", size = 79886, upload-time = "2025-10-17T06:19:15.345Z" }, + { url = "https://files.pythonhosted.org/packages/d5/db/1a7cb6757a1e32376fa2dfce00c815ea4ee614a94f9bff8228e37420c183/crc32c-2.8-cp312-cp312-win32.whl", hash = "sha256:a73d03ce3604aa5d7a2698e9057a0eef69f529c46497b27ee1c38158e90ceb76", size = 64896, upload-time = "2025-10-17T06:19:16.457Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8e/2024de34399b2e401a37dcb54b224b56c747b0dc46de4966886827b4d370/crc32c-2.8-cp312-cp312-win_amd64.whl", hash = "sha256:56b3b7d015247962cf58186e06d18c3d75a1a63d709d3233509e1c50a2d36aa2", size = 66645, upload-time = "2025-10-17T06:19:17.235Z" }, + { url = "https://files.pythonhosted.org/packages/e8/d8/3ae227890b3be40955a7144106ef4dd97d6123a82c2a5310cdab58ca49d8/crc32c-2.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:36f1e03ee9e9c6938e67d3bcb60e36f260170aa5f37da1185e04ef37b56af395", size = 66380, upload-time = "2025-10-17T06:19:18.009Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8b/178d3f987cd0e049b484615512d3f91f3d2caeeb8ff336bb5896ae317438/crc32c-2.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b2f3226b94b85a8dd9b3533601d7a63e9e3e8edf03a8a169830ee8303a199aeb", size = 63048, upload-time = "2025-10-17T06:19:18.853Z" }, + { url = "https://files.pythonhosted.org/packages/f2/a1/48145ae2545ebc0169d3283ebe882da580ea4606bfb67cf4ca922ac3cfc3/crc32c-2.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6e08628bc72d5b6bc8e0730e8f142194b610e780a98c58cb6698e665cb885a5b", size = 61530, upload-time = "2025-10-17T06:19:19.974Z" }, + { url = "https://files.pythonhosted.org/packages/06/4b/cf05ed9d934cc30e5ae22f97c8272face420a476090e736615d9a6b53de0/crc32c-2.8-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:086f64793c5ec856d1ab31a026d52ad2b895ac83d7a38fce557d74eb857f0a82", size = 80001, upload-time = "2025-10-17T06:19:20.784Z" }, + { url = "https://files.pythonhosted.org/packages/15/ab/4b04801739faf36345f6ba1920be5b1c70282fec52f8280afd3613fb13e2/crc32c-2.8-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcf72ee7e0135b3d941c34bb2c26c3fc6bc207106b49fd89aaafaeae223ae209", size = 81543, upload-time = "2025-10-17T06:19:21.557Z" }, + { url = "https://files.pythonhosted.org/packages/a9/1b/6e38dde5bfd2ea69b7f2ab6ec229fcd972a53d39e2db4efe75c0ac0382ce/crc32c-2.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a717dd9c3fd777d9bc6603717eae172887d402c4ab589d124ebd0184a83f89e", size = 80644, upload-time = "2025-10-17T06:19:22.325Z" }, + { url = "https://files.pythonhosted.org/packages/ce/45/012176ffee90059ae8ec7131019c71724ea472aa63e72c0c8edbd1fad1d7/crc32c-2.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0450bb845b3c3c7b9bdc0b4e95620ec9a40824abdc8c86d6285c919a90743c1a", size = 79919, upload-time = "2025-10-17T06:19:23.101Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/f557629842f9dec2b3461cb3a0d854bb586ec45b814cea58b082c32f0dde/crc32c-2.8-cp313-cp313-win32.whl", hash = "sha256:765d220bfcbcffa6598ac11eb1e10af0ee4802b49fe126aa6bf79f8ddb9931d1", size = 64896, upload-time = "2025-10-17T06:19:23.88Z" }, + { url = "https://files.pythonhosted.org/packages/d0/db/fd0f698c15d1e21d47c64181a98290665a08fcbb3940cd559e9c15bda57e/crc32c-2.8-cp313-cp313-win_amd64.whl", hash = "sha256:171ff0260d112c62abcce29332986950a57bddee514e0a2418bfde493ea06bb3", size = 66646, upload-time = "2025-10-17T06:19:24.702Z" }, + { url = "https://files.pythonhosted.org/packages/db/b9/8e5d7054fe8e7eecab10fd0c8e7ffb01439417bdb6de1d66a81c38fc4a20/crc32c-2.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b977a32a3708d6f51703c8557008f190aaa434d7347431efb0e86fcbe78c2a50", size = 66203, upload-time = "2025-10-17T06:19:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/55/5f/cc926c70057a63cc0c98a3c8a896eb15fc7e74d3034eadd53c94917c6cc3/crc32c-2.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7399b01db4adaf41da2fb36fe2408e75a8d82a179a9564ed7619412e427b26d6", size = 62956, upload-time = "2025-10-17T06:19:26.652Z" }, + { url = "https://files.pythonhosted.org/packages/a1/8a/0660c44a2dd2cb6ccbb529eb363b9280f5c766f1017bc8355ed8d695bd94/crc32c-2.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4379f73f9cdad31958a673d11a332ec725ca71572401ca865867229f5f15e853", size = 61442, upload-time = "2025-10-17T06:19:27.74Z" }, + { url = "https://files.pythonhosted.org/packages/f5/5a/6108d2dfc0fe33522ce83ba07aed4b22014911b387afa228808a278e27cd/crc32c-2.8-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e68264555fab19bab08331550dab58573e351a63ed79c869d455edd3b0aa417", size = 79109, upload-time = "2025-10-17T06:19:28.535Z" }, + { url = "https://files.pythonhosted.org/packages/84/1e/c054f9e390090c197abf3d2936f4f9effaf0c6ee14569ae03d6ddf86958a/crc32c-2.8-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b48f2486727b8d0e7ccbae4a34cb0300498433d2a9d6b49cb13cb57c2e3f19cb", size = 80987, upload-time = "2025-10-17T06:19:29.305Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ad/1650e5c3341e4a485f800ea83116d72965030c5d48ccc168fcc685756e4d/crc32c-2.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ecf123348934a086df8c8fde7f9f2d716d523ca0707c5a1367b8bb00d8134823", size = 79994, upload-time = "2025-10-17T06:19:30.109Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3b/f2ed924b177729cbb2ab30ca2902abff653c31d48c95e7b66717a9ca9fcc/crc32c-2.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e636ac60f76de538f7a2c0d0f3abf43104ee83a8f5e516f6345dc283ed1a4df7", size = 79046, upload-time = "2025-10-17T06:19:30.894Z" }, + { url = "https://files.pythonhosted.org/packages/4b/80/413b05ee6ace613208b31b3670c3135ee1cf451f0e72a9c839b4946acc04/crc32c-2.8-cp313-cp313t-win32.whl", hash = "sha256:8dd4a19505e0253892e1b2f1425cc3bd47f79ae5a04cb8800315d00aad7197f2", size = 64837, upload-time = "2025-10-17T06:19:32.03Z" }, + { url = "https://files.pythonhosted.org/packages/3b/1b/85eddb6ac5b38496c4e35c20298aae627970c88c3c624a22ab33e84f16c7/crc32c-2.8-cp313-cp313t-win_amd64.whl", hash = "sha256:4bb18e4bd98fb266596523ffc6be9c5b2387b2fa4e505ec56ca36336f49cb639", size = 66574, upload-time = "2025-10-17T06:19:33.143Z" }, + { url = "https://files.pythonhosted.org/packages/aa/df/50e9079b532ff53dbfc0e66eed781374bd455af02ed5df8b56ad538de4ff/crc32c-2.8-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3a3b2e4bcf7b3ee333050e7d3ff38e2ba46ea205f1d73d8949b248aaffe937ac", size = 66399, upload-time = "2025-10-17T06:19:34.279Z" }, + { url = "https://files.pythonhosted.org/packages/5a/2e/67e3b0bc3d30e46ea5d16365cc81203286387671e22f2307eb41f19abb9c/crc32c-2.8-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:445e559e66dff16be54f8a4ef95aa6b01db799a639956d995c5498ba513fccc2", size = 63044, upload-time = "2025-10-17T06:19:35.062Z" }, + { url = "https://files.pythonhosted.org/packages/36/ea/1723b17437e4344ed8d067456382ecb1f5b535d83fdc5aaebab676c6d273/crc32c-2.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bf3040919e17afa5782e01b1875d6a05f44b8f19c05f211d8b9f8a1deb8bbd9c", size = 61541, upload-time = "2025-10-17T06:19:36.204Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6a/cbec8a235c5b46a01f319939b538958662159aec0ed3a74944e3a6de21f1/crc32c-2.8-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5607ab8221e1ffd411f64aa40dbb6850cf06dd2908c9debd05d371e1acf62ff3", size = 80139, upload-time = "2025-10-17T06:19:37.351Z" }, + { url = "https://files.pythonhosted.org/packages/21/31/d096722fe74b692d6e8206c27da1ea5f6b2a12ff92c54a62a6ba2f376254/crc32c-2.8-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f5db4f16816926986d3c94253314920689706ae13a9bf4888b47336c6735ce", size = 81736, upload-time = "2025-10-17T06:19:38.16Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a2/f75ef716ff7e3c22f385ba6ef30c5de80c19a21ebe699dc90824a1903275/crc32c-2.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:70b0153c4d418b673309d3529334d117e1074c4a3b2d7f676e430d72c14de67b", size = 80795, upload-time = "2025-10-17T06:19:38.948Z" }, + { url = "https://files.pythonhosted.org/packages/d8/94/6d647a12d96ab087d9b8eacee3da073f981987827d57c7072f89ffc7b6cd/crc32c-2.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c8933531442042438753755a5c8a9034e4d88b01da9eb796f7e151b31a7256c", size = 80042, upload-time = "2025-10-17T06:19:39.725Z" }, + { url = "https://files.pythonhosted.org/packages/cd/dc/32b8896b40a0afee7a3c040536d0da5a73e68df2be9fadd21770fd158e16/crc32c-2.8-cp314-cp314-win32.whl", hash = "sha256:cdc83a3fe6c4e5df9457294cfd643de7d95bd4e9382c1dd6ed1e0f0f9169172c", size = 64914, upload-time = "2025-10-17T06:19:40.527Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b4/4308b27d307e8ecaf8dd1dcc63bbb0e47ae1826d93faa3e62d1ee00ee2d5/crc32c-2.8-cp314-cp314-win_amd64.whl", hash = "sha256:509e10035106df66770fe24b9eb8d9e32b6fb967df17744402fb67772d8b2bc7", size = 66723, upload-time = "2025-10-17T06:19:42.449Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/a19d2489fa997a143bfbbf971a5c9a43f8b1ba9e775b1fb362d8fb15260c/crc32c-2.8-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:864359a39777a07b09b28eb31337c0cc603d5c1bf0fc328c3af736a8da624ec0", size = 66201, upload-time = "2025-10-17T06:19:43.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/c2/5f82f22d2c1242cb6f6fe92aa9a42991ebea86de994b8f9974d9c1d128e2/crc32c-2.8-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:14511d7cfc5d9f5e1a6c6b64caa6225c2bdc1ed00d725e9a374a3e84073ce180", size = 62956, upload-time = "2025-10-17T06:19:44.099Z" }, + { url = "https://files.pythonhosted.org/packages/9b/61/3d43d33489cf974fb78bfb3500845770e139ae6d1d83473b660bd8f79a6c/crc32c-2.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:918b7999b52b5dcbcea34081e9a02d46917d571921a3f209956a9a429b2e06e5", size = 61443, upload-time = "2025-10-17T06:19:44.89Z" }, + { url = "https://files.pythonhosted.org/packages/52/6d/f306ce64a352a3002f76b0fc88a1373f4541f9d34fad3668688610bab14b/crc32c-2.8-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cc445da03fc012a5a03b71da1df1b40139729e6a5571fd4215ab40bfb39689c7", size = 79106, upload-time = "2025-10-17T06:19:45.688Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b7/1f74965dd7ea762954a69d172dfb3a706049c84ffa45d31401d010a4a126/crc32c-2.8-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e3dde2ec59a8a830511d72a086ead95c0b0b7f0d418f93ea106244c5e77e350", size = 80983, upload-time = "2025-10-17T06:19:46.792Z" }, + { url = "https://files.pythonhosted.org/packages/1b/50/af93f0d91ccd61833ce77374ebfbd16f5805f5c17d18c6470976d9866d76/crc32c-2.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:61d51681a08b6a2a2e771b7f0cd1947fb87cb28f38ed55a01cb7c40b2ac4cdd8", size = 80009, upload-time = "2025-10-17T06:19:47.619Z" }, + { url = "https://files.pythonhosted.org/packages/ee/fa/94f394beb68a88258af694dab2f1284f55a406b615d7900bdd6235283bc4/crc32c-2.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:67c0716c3b1a02d5235be649487b637eed21f2d070f2b3f63f709dcd2fefb4c7", size = 79066, upload-time = "2025-10-17T06:19:48.409Z" }, + { url = "https://files.pythonhosted.org/packages/91/c6/a6050e0c64fd73c67a97da96cb59f08b05111e00b958fb87ecdce99f17ac/crc32c-2.8-cp314-cp314t-win32.whl", hash = "sha256:2e8fe863fbbd8bdb6b414a2090f1b0f52106e76e9a9c96a413495dbe5ebe492a", size = 64869, upload-time = "2025-10-17T06:19:49.197Z" }, + { url = "https://files.pythonhosted.org/packages/08/1f/c7735034e401cb1ea14f996a224518e3a3fa9987cb13680e707328a7d779/crc32c-2.8-cp314-cp314t-win_amd64.whl", hash = "sha256:20a9cfb897693eb6da19e52e2a7be2026fd4d9fc8ae318f086c0d71d5dd2d8e0", size = 66633, upload-time = "2025-10-17T06:19:50.003Z" }, + { url = "https://files.pythonhosted.org/packages/a7/1d/dd926c68eb8aac8b142a1a10b8eb62d95212c1cf81775644373fe7cceac2/crc32c-2.8-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5833f4071da7ea182c514ba17d1eee8aec3c5be927d798222fbfbbd0f5eea02c", size = 62345, upload-time = "2025-10-17T06:20:09.39Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/803404e5abea2ef2c15042edca04bbb7f625044cca879e47f186b43887c2/crc32c-2.8-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1dc4da036126ac07b39dd9d03e93e585ec615a2ad28ff12757aef7de175295a8", size = 61229, upload-time = "2025-10-17T06:20:10.236Z" }, + { url = "https://files.pythonhosted.org/packages/fc/3a/00cc578cd27ed0b22c9be25cef2c24539d92df9fa80ebd67a3fc5419724c/crc32c-2.8-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:15905fa78344654e241371c47e6ed2411f9eeb2b8095311c68c88eccf541e8b4", size = 64108, upload-time = "2025-10-17T06:20:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/6b/bc/0587ef99a1c7629f95dd0c9d4f3d894de383a0df85831eb16c48a6afdae4/crc32c-2.8-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c596f918688821f796434e89b431b1698396c38bf0b56de873621528fe3ecb1e", size = 64815, upload-time = "2025-10-17T06:20:11.919Z" }, + { url = "https://files.pythonhosted.org/packages/73/42/94f2b8b92eae9064fcfb8deef2b971514065bd606231f8857ff8ae02bebd/crc32c-2.8-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8d23c4fe01b3844cb6e091044bc1cebdef7d16472e058ce12d9fadf10d2614af", size = 66659, upload-time = "2025-10-17T06:20:12.766Z" }, +] + [[package]] name = "cryptography" version = "49.0.0" @@ -803,7 +901,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" }, @@ -838,37 +936,37 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, ] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "sys_platform == 'linux'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "sys_platform == 'linux'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cusolver", marker = "sys_platform == 'linux'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "sys_platform == 'linux'" }, ] [[package]] @@ -1027,7 +1125,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1441,20 +1539,21 @@ name = "ibm-watsonx-ai" version = "1.3.42" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "cachetools" }, - { name = "certifi" }, - { name = "httpx" }, - { name = "ibm-cos-sdk" }, - { name = "lomond" }, - { name = "packaging" }, - { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" } }, - { name = "requests" }, - { name = "tabulate" }, - { name = "urllib3" }, + { name = "cachetools", marker = "python_full_version < '3.11'" }, + { name = "certifi", marker = "python_full_version < '3.11'" }, + { name = "httpx", marker = "python_full_version < '3.11'" }, + { name = "ibm-cos-sdk", marker = "python_full_version < '3.11'" }, + { name = "lomond", marker = "python_full_version < '3.11'" }, + { name = "packaging", marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "requests", marker = "python_full_version < '3.11'" }, + { name = "tabulate", marker = "python_full_version < '3.11'" }, + { name = "urllib3", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c7/56/2e3df38a1f13062095d7bde23c87a92f3898982993a15186b1bfecbd206f/ibm_watsonx_ai-1.3.42.tar.gz", hash = "sha256:ee5be59009004245d957ce97d1227355516df95a2640189749487614fef674ff", size = 688651, upload-time = "2025-10-01T13:35:41.527Z" } wheels = [ @@ -1468,25 +1567,28 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "cachetools" }, - { name = "certifi" }, - { name = "httpx" }, - { name = "ibm-cos-sdk" }, - { name = "lomond" }, - { name = "packaging" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, + { name = "cachetools", marker = "python_full_version >= '3.11'" }, + { name = "certifi", marker = "python_full_version >= '3.11'" }, + { name = "httpx", marker = "python_full_version >= '3.11'" }, + { name = "ibm-cos-sdk", marker = "python_full_version >= '3.11'" }, + { name = "lomond", marker = "python_full_version >= '3.11'" }, + { name = "packaging", marker = "python_full_version >= '3.11'" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, - { name = "requests" }, - { name = "tabulate" }, - { name = "urllib3" }, + { name = "requests", marker = "python_full_version >= '3.11'" }, + { name = "tabulate", marker = "python_full_version >= '3.11'" }, + { name = "urllib3", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/a3/c756b534696ab2f3f29882fdb7ca7198b7a5c94e10c0a3a327853d6d6b79/ibm_watsonx_ai-1.5.14.tar.gz", hash = "sha256:a756488bd57e87c0fc51be42dcba871143cfe0ac1e805c497c5047e1e4f13e9d", size = 735804, upload-time = "2026-06-22T12:32:43.85Z" } wheels = [ @@ -1559,21 +1661,22 @@ name = "ipython" version = "8.39.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "exceptiongroup" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions" }, + { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.11'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "jedi", marker = "python_full_version < '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, + { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, + { name = "pygments", marker = "python_full_version < '3.11'" }, + { name = "stack-data", marker = "python_full_version < '3.11'" }, + { name = "traitlets", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1587,26 +1690,29 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "decorator" }, - { name = "ipython-pygments-lexers" }, - { name = "jedi" }, - { name = "matplotlib-inline" }, - { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit" }, - { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments" }, - { name = "stack-data" }, - { name = "traitlets" }, - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version >= '3.11'" }, + { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, + { name = "jedi", marker = "python_full_version >= '3.11'" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, + { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, + { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "stack-data", marker = "python_full_version >= '3.11'" }, + { name = "traitlets", marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -1618,7 +1724,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments" }, + { name = "pygments", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1935,6 +2041,7 @@ embeddings = [ { name = "ibm-watsonx-ai", version = "1.3.42", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "ibm-watsonx-ai", version = "1.5.14", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "instructorembedding" }, + { name = "oci" }, { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, @@ -1949,7 +2056,7 @@ otel = [ ] pylance = [ { name = "pylance", version = "15.0.0b4", source = { registry = "https://pypi.fury.io/lance-format/" }, marker = "sys_platform == 'darwin'" }, - { name = "pylance", version = "15.0.0b7", source = { registry = "https://pypi.fury.io/lance-format/" }, marker = "sys_platform != 'darwin'" }, + { name = "pylance", version = "15.0.0b9", source = { registry = "https://pypi.fury.io/lance-format/" }, marker = "sys_platform != 'darwin'" }, ] siglip = [ { name = "pillow" }, @@ -1970,7 +2077,7 @@ tests = [ { name = "pyarrow" }, { name = "pyarrow-stubs" }, { name = "pylance", version = "15.0.0b4", source = { registry = "https://pypi.fury.io/lance-format/" }, marker = "sys_platform == 'darwin'" }, - { name = "pylance", version = "15.0.0b7", source = { registry = "https://pypi.fury.io/lance-format/" }, marker = "sys_platform != 'darwin'" }, + { name = "pylance", version = "15.0.0b9", source = { registry = "https://pypi.fury.io/lance-format/" }, marker = "sys_platform != 'darwin'" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-mock" }, @@ -2002,6 +2109,7 @@ requires-dist = [ { name = "mkdocs-material", marker = "extra == 'docs'" }, { name = "mkdocstrings-python", marker = "extra == 'docs'" }, { name = "numpy", specifier = ">=1.24.0" }, + { name = "oci", marker = "extra == 'embeddings'", specifier = ">=2.165.1" }, { name = "ollama", marker = "extra == 'embeddings'", specifier = ">=0.3.0" }, { name = "open-clip-torch", marker = "extra == 'clip'" }, { name = "open-clip-torch", marker = "extra == 'embeddings'", specifier = ">=2.20.0" }, @@ -2021,7 +2129,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "sys_platform == 'darwin' and extra == 'tests'", specifier = "==15.0.0b4" }, - { name = "pylance", marker = "sys_platform != 'darwin' and extra == 'tests'", specifier = "==15.0.0b7" }, + { name = "pylance", marker = "sys_platform != 'darwin' and extra == 'tests'", specifier = "==15.0.0b9" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=15.0.0b3" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, @@ -2622,7 +2730,8 @@ name = "networkx" version = "3.4.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368, upload-time = "2024-10-21T12:39:38.695Z" } @@ -2637,11 +2746,14 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } @@ -2663,7 +2775,8 @@ name = "numpy" version = "2.2.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } @@ -2729,7 +2842,8 @@ name = "numpy" version = "2.4.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } @@ -2814,9 +2928,11 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } @@ -2907,7 +3023,7 @@ name = "nvidia-cudnn-cu13" version = "9.19.0.56" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, @@ -2919,7 +3035,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -2949,9 +3065,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -2963,7 +3079,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -3015,6 +3131,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] +[[package]] +name = "oci" +version = "2.188.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "certifi" }, + { name = "circuitbreaker" }, + { name = "crc32c" }, + { name = "cryptography" }, + { name = "pyjwt" }, + { name = "pyopenssl" }, + { name = "python-dateutil" }, + { name = "python-pkcs11" }, + { name = "pytz" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/09/69de33858a8574f6e7f7ebf3d027ac198986c95d3f0efd5c203a882be8d6/oci-2.188.0.tar.gz", hash = "sha256:a4ff307ed315112cf357d9dc0bd5091cb0bae77892870b51ffd0cf46175ba8b6", size = 18377765, upload-time = "2026-10-13T03:51:53.525Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/1b/3138ece59f7ff4afbf4ad53c9d2f88ec7e693f8eae893ebac305e31b45f4/oci-2.188.0-py3-none-any.whl", hash = "sha256:97cf5a0253b45f758c22070c009b7c3197d605d0711a64c7bb060e3d67eda127", size = 37373455, upload-time = "2026-10-13T03:51:47.162Z" }, +] + [[package]] name = "ollama" version = "0.6.2" @@ -3137,14 +3275,15 @@ name = "pandas" version = "2.2.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "python-dateutil", marker = "python_full_version < '3.11'" }, + { name = "pytz", marker = "python_full_version < '3.11'" }, + { name = "tzdata", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" } wheels = [ @@ -3189,17 +3328,19 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "python-dateutil", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, + { name = "pytz", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, + { name = "tzdata", marker = "python_full_version >= '3.11' and python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -3259,13 +3400,14 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.14'" }, + { name = "tzdata", marker = "(python_full_version >= '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.14' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -3373,7 +3515,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -3974,11 +4116,11 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "lance-namespace" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pyarrow" }, + { name = "lance-namespace", marker = "sys_platform == 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform == 'darwin'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform == 'darwin'" }, + { name = "pyarrow", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://pypi.fury.io/lance-format/-/ver_1Bbzay/pylance-15.0.0b4-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c948ac22105a0bd9f8314edb883edac421a64b059d992ea6d614773cd2fa2e65" }, @@ -3986,29 +4128,33 @@ wheels = [ [[package]] name = "pylance" -version = "15.0.0b7" +version = "15.0.0b9" source = { registry = "https://pypi.fury.io/lance-format/" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", ] dependencies = [ - { name = "lance-namespace" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "pyarrow" }, + { name = "lance-namespace", marker = "sys_platform != 'darwin'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*' and sys_platform != 'darwin'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and sys_platform != 'darwin'" }, + { name = "pyarrow", marker = "sys_platform != 'darwin'" }, ] wheels = [ - { url = "https://pypi.fury.io/lance-format/-/ver_Os2g1/pylance-15.0.0b7-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ce5243c8d6fabc48775cd93b98a0506c00cc0cf6f99c59b159561d4aa4d3a1b" }, - { url = "https://pypi.fury.io/lance-format/-/ver_1mFdl8/pylance-15.0.0b7-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d857573666e278dcb21e32982596cdacf188cc4c8e7a8440f8794e7068301a8" }, - { url = "https://pypi.fury.io/lance-format/-/ver_29wiYw/pylance-15.0.0b7-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:e3c99911f7b84cb4eeba61bb1d396a8e5e31a79fee879dd41f396d83e4ca992d" }, - { url = "https://pypi.fury.io/lance-format/-/ver_15Kfqv/pylance-15.0.0b7-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:dbe9f88fa69971feef0a68f1d352e4c26c9f31c5f4885b41929a9af89eab677a" }, - { url = "https://pypi.fury.io/lance-format/-/ver_1DaQkG/pylance-15.0.0b7-cp310-abi3-win_amd64.whl", hash = "sha256:638907fed3d959e0852c0e5799323551fd1778cbd4ac035273a5427718d9daac" }, + { url = "https://pypi.fury.io/lance-format/-/ver_11uz9g/pylance-15.0.0b9-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:749d4da73a22f5d33f6926387109dfaf814413d3bd1c36b974edf81ca040eb04" }, + { url = "https://pypi.fury.io/lance-format/-/ver_Zl1Qd/pylance-15.0.0b9-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c49523c68122f4cb42fd310155af6f420f90559752f0735d1503c06f7f45629" }, + { url = "https://pypi.fury.io/lance-format/-/ver_23GhEu/pylance-15.0.0b9-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:78ff23c91b1b4113bae7715fd143af1fec732cdc0fd2429db6d3149de16d565a" }, + { url = "https://pypi.fury.io/lance-format/-/ver_1IKCVE/pylance-15.0.0b9-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ee8f71b7bdf16bb44305f961b6a2cf4a75f47f1db7d217955e2aca0020acdf05" }, + { url = "https://pypi.fury.io/lance-format/-/ver_10hNTp/pylance-15.0.0b9-cp310-abi3-win_amd64.whl", hash = "sha256:28f1f92c8af8072a7fdc6f4eca5a8814a54e457843e2ac7e0f00554da5152d90" }, ] [[package]] @@ -4024,6 +4170,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d6/54/da572c98c0b77626a91b5d3b89f0231d8bff5125c225420908632f8b342d/pymdown_extensions-11.0.1-py3-none-any.whl", hash = "sha256:db3943a62bab7e03af1364f0c4083e64b91fb097675a4b6cceccfbe9a77e5eb2", size = 269455, upload-time = "2026-07-02T17:59:21.271Z" }, ] +[[package]] +name = "pyopenssl" +version = "26.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, +] + [[package]] name = "pyright" version = "1.1.411" @@ -4106,6 +4265,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/78/9b77ecb4644d1bbea94d29abf78f21c47eca6eb79e9745b702ec0bed2e19/python_discovery-1.4.3-py3-none-any.whl", hash = "sha256:b6e1e4a7d9e3f6948c39746ffe8218225162d738ba39d05ab1d2f6c1cac4878c", size = 33885, upload-time = "2026-07-03T13:21:50.174Z" }, ] +[[package]] +name = "python-pkcs11" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/33/4287ebdcbdcae6f93521be56ffdd5f474b6aebda1a50df4c4b9870a140fa/python_pkcs11-0.9.4.tar.gz", hash = "sha256:3d950aadefa473b880dc75f01539133487e65ec2a97db1e955c4f6bef1bd71d5", size = 180658, upload-time = "2026-04-13T13:32:51.769Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/98/6da01b81312c481f743b675577ae64ee1887a7d1c6ca4294b136a189f581/python_pkcs11-0.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fbf1436684a74b2b26cb5f51e970f2505d81bac74ee084641b625d956f08af29", size = 558896, upload-time = "2026-04-13T13:31:51.089Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b3/ffa8d044bf9831c20a7a9e305117cde8512c6ccdf26d5e95a605dc7bc5c6/python_pkcs11-0.9.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2b1af32dad1457250959d42093e05ebc98f94f62d4c2393d72eef0c73c846c7", size = 1833458, upload-time = "2026-04-13T13:31:52.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f7/d73bbfc9cca5a6837b7fce2aebf4a71290bacbcf31709271ff81faa92744/python_pkcs11-0.9.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ac2f25bc27b618f8f1d17b4d6cd5887e4202ee32edec206052d6f46943f8990", size = 1851767, upload-time = "2026-04-13T13:31:54.673Z" }, + { url = "https://files.pythonhosted.org/packages/5d/b6/08a12c496ec6e49a40afc9aaad526b110a99e778822295778359fc2a1161/python_pkcs11-0.9.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a3edf4ae24def328f454d38dffef1677baec69d376755523d4e56280fc177173", size = 1779639, upload-time = "2026-04-13T13:31:56.744Z" }, + { url = "https://files.pythonhosted.org/packages/bd/78/1792a808dd3aeeb46acfdcd2d28084d22c7648ce7b7cb925fc1c78b07052/python_pkcs11-0.9.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:975389457f533bdbaa96b845e9517ef6c4cdaed627a54515fb77165014399c27", size = 1830315, upload-time = "2026-04-13T13:31:58.715Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/80063318f6fc8f775200add5eefb0a2b6dcf6bdc5da89b562c8d5530cb9c/python_pkcs11-0.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:8ed0225ae59ec061575076bf67db4bd2b4b71e1157462fed4c7c10f11bd6709d", size = 284002, upload-time = "2026-04-13T13:32:00.268Z" }, + { url = "https://files.pythonhosted.org/packages/00/10/e2f33f3acca84b72d36b7bc66f62f2eac9b12b61268fb1a3c452ca084fcf/python_pkcs11-0.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:585bfeaafd12db0875fa74e2c4cbe8b055a75a6c359f873a4777deefbcbb2c6f", size = 557105, upload-time = "2026-04-13T13:32:01.725Z" }, + { url = "https://files.pythonhosted.org/packages/db/e6/fd48a495d57f488976b0871f1c08efed1a19722aadea253a49b989c611b1/python_pkcs11-0.9.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8bff8267d1e5813c591290b02c1c7e700786b94a3ba750fda5f8d44f9fc7191", size = 1916836, upload-time = "2026-04-13T13:32:03.486Z" }, + { url = "https://files.pythonhosted.org/packages/bc/d2/a627fdd228c1253e25f9dd4b6facf3499cb7b2d76260bf337b1f84cb1bd2/python_pkcs11-0.9.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54399a5235c427c25f654c89f451648084dbabe0696bf9e685e86c741460857b", size = 1935363, upload-time = "2026-04-13T13:32:05.101Z" }, + { url = "https://files.pythonhosted.org/packages/3c/c0/3430e1d3cdd1f684cc9766030867d850fd75fd810c4729d7b93b6fed3648/python_pkcs11-0.9.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b1e9d98d5b4f732eba6503260afe805053fbd67663bd00184cdc8c478f26625a", size = 1865096, upload-time = "2026-04-13T13:32:06.759Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/1afe8d791787e00212d35895919328e8d6381379f5421f5f1cd5897aa089/python_pkcs11-0.9.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:afe3d78afa6cf30c6a26bfe4e9215500809c3b76f7561d809100e367d7a22dd4", size = 1914261, upload-time = "2026-04-13T13:32:08.923Z" }, + { url = "https://files.pythonhosted.org/packages/58/f4/7b207d44a1f21352018225396e8b8e4dc7763c4975eecfcfa27375188bba/python_pkcs11-0.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:ed40fc57d78714e3281da4e6ee67fb8aebe677a83c9430af3a1d5defa16e9201", size = 284870, upload-time = "2026-04-13T13:32:10.731Z" }, + { url = "https://files.pythonhosted.org/packages/f4/af/1b6525ba52b7216e28fd2ccc0802fd1382a5f672c8d86e7f5dd0feff164c/python_pkcs11-0.9.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:aa22cd0d172a560618134943784ee6a405b578d187b41d7cc66dc43d8f482dbf", size = 523923, upload-time = "2026-04-13T13:32:12.145Z" }, + { url = "https://files.pythonhosted.org/packages/ad/6a/c199375fdda4e5ac1e630f05500f528c651d2377c48e0ecaa0d13fefd93c/python_pkcs11-0.9.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e05cb5d780bbccdee2588be4fc98c977e4f2024a70ae70a1a109d47a9afd525b", size = 1847563, upload-time = "2026-04-13T13:32:13.803Z" }, + { url = "https://files.pythonhosted.org/packages/71/54/fe792b7ce715866c210a9263464e776dc28c1b8424d9c2e96b6f6cc3daeb/python_pkcs11-0.9.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f49715b81b7afaaa8efaa9d91ed0c1eaac34f190a908347e42d4530da1bc5eb", size = 1895189, upload-time = "2026-04-13T13:32:15.763Z" }, + { url = "https://files.pythonhosted.org/packages/d0/5d/e30b3664794c5d49e995831a536d456a92c283eb73e5c933226d83ad3125/python_pkcs11-0.9.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:050ee672467aa885de79ad4f1f57e4a16f2bca9b4fcac42c72e14e31d5ba700f", size = 1780227, upload-time = "2026-04-13T13:32:17.632Z" }, + { url = "https://files.pythonhosted.org/packages/79/7a/f781c7360eecc96e8331cb9e8c38a43c7bcfcb03b075ef520cd8575c76ff/python_pkcs11-0.9.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2bbab746a84f15a0727e017289e074a322c061cf0203fa2706729bde9ac60648", size = 1854335, upload-time = "2026-04-13T13:32:19.181Z" }, + { url = "https://files.pythonhosted.org/packages/ea/36/e3652bbaa37fde1444ee861e82448d351f89bb8672577394bbe9c8a4cde6/python_pkcs11-0.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:91fe4e11c7187e548293dc5c020f69665304ebdaa5724b7806251c6473d70004", size = 274412, upload-time = "2026-04-13T13:32:20.762Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a9/52150a1bd0c01d16e73de557fc25aa3d682c4bda6ee6931fabfbaf681550/python_pkcs11-0.9.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:38e80866d35e3d591972af796d2fd2557a4071b1586d4f79e14d93ab61e60425", size = 521406, upload-time = "2026-04-13T13:32:22.107Z" }, + { url = "https://files.pythonhosted.org/packages/34/88/e0eb7702d369f9844df6a341f5f31ecade2f5463a32290fb291270d0c766/python_pkcs11-0.9.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dcb7f6e08a5c2a9a0c6f557b829bba12dd4ae6489d2d65e92cea222650ca3160", size = 1852110, upload-time = "2026-04-13T13:32:23.47Z" }, + { url = "https://files.pythonhosted.org/packages/48/44/ada8dd48e7fc4db5d5f715b9206bdf8bbbe756592fe0206574c8f68253a0/python_pkcs11-0.9.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f700992fe99a147fc64a6003999615337e5e1fe27c5f381b0d91c1a68db6967", size = 1889010, upload-time = "2026-04-13T13:32:24.989Z" }, + { url = "https://files.pythonhosted.org/packages/07/36/4ebdc96f660228dc0810db4b81db6fbad6a929656c130ef8fd366cebf275/python_pkcs11-0.9.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3680ad0ac2a48dddd5ffdc87d2bfefe5a1995f8db26d5aa69e985771b5a2ab66", size = 1790342, upload-time = "2026-04-13T13:32:26.836Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/8850277a94b434918c742c4a55704c5d105fe94b88201906624a69b298a5/python_pkcs11-0.9.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3912989be7e9c10c330fe13a663baeab568f222addbe612e4183409b20bffc36", size = 1852955, upload-time = "2026-04-13T13:32:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0c/f75061daf8ae64b340235f6afe206a0d23f148c0e3de98d384ecd7527d8d/python_pkcs11-0.9.4-cp313-cp313-win_amd64.whl", hash = "sha256:2d2b5827fb888fc388c2026e040c14a257c41ba97d543fbf08b5730aad0c6899", size = 275081, upload-time = "2026-04-13T13:32:30.603Z" }, + { url = "https://files.pythonhosted.org/packages/e8/04/aa4c33e3d14612d3855dd3f6f72f6ef9db82d1575a387fce8a22678fe908/python_pkcs11-0.9.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0e1c88481eeada4fd81cfefe117413137a2868cc072e588c05972aca92020cc8", size = 524552, upload-time = "2026-04-13T13:32:32.124Z" }, + { url = "https://files.pythonhosted.org/packages/a3/fd/eaf6527c25c116a29fec54776634c1afc7cb9326a2e07c1af1f9f843494e/python_pkcs11-0.9.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5c24e6a25ee6756f8116960a9b0757c54dc355ae1b86ec47a0704bde5c6ee62", size = 1847496, upload-time = "2026-04-13T13:32:33.562Z" }, + { url = "https://files.pythonhosted.org/packages/82/d7/711f205660a4a6ce4f2e937b2fe1bd9f1274449658a07cd050cfb4ed9507/python_pkcs11-0.9.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2351da910df98e6b9f4befdafbfc2c81fc9c86358ec88dd802e67e6e3ae85f33", size = 1865861, upload-time = "2026-04-13T13:32:35.23Z" }, + { url = "https://files.pythonhosted.org/packages/de/51/43be836fb48a87e8589adddd47fbee86c0b0c09b9edd60ea267cc7fefc55/python_pkcs11-0.9.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7081ec0a3456a40abcde382c8b64048b922076391e9ae90a272a970c96dc10c8", size = 1783570, upload-time = "2026-04-13T13:32:37.357Z" }, + { url = "https://files.pythonhosted.org/packages/b3/05/492e46d0f60bccebed71f4e3429a55842ad5989e07d9ad98989d762d9455/python_pkcs11-0.9.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:16aba462e30169a9898f51532297aad94f13ee944873fa48eb2504b6efaf373d", size = 1830078, upload-time = "2026-04-13T13:32:38.897Z" }, + { url = "https://files.pythonhosted.org/packages/53/ef/4b95b463e48fd2474e6c51446ea45d9b7042b8a67b5abc1e06bfbfbc8595/python_pkcs11-0.9.4-cp314-cp314-win_amd64.whl", hash = "sha256:d13f9a6b76d56ff5628a85530aab7b021e9f90e3300be985423b69301c6aae38", size = 280929, upload-time = "2026-04-13T13:32:40.712Z" }, +] + [[package]] name = "pytz" version = "2026.2" @@ -4433,7 +4633,8 @@ name = "rpds-py" version = "0.30.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } @@ -4561,11 +4762,14 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } @@ -4765,14 +4969,15 @@ name = "scikit-learn" version = "1.7.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -4815,21 +5020,24 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "joblib", marker = "python_full_version >= '3.11'" }, + { name = "narwhals", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -4870,11 +5078,12 @@ name = "scipy" version = "1.15.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version < '3.11' and sys_platform != 'darwin'", + "python_full_version < '3.11' and sys_platform == 'linux'", + "python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -4930,11 +5139,12 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.11.*' and sys_platform != 'darwin'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -5007,13 +5217,15 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", - "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.14' and sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32'", "python_full_version >= '3.14' and sys_platform == 'darwin'", - "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'linux'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'darwin' and sys_platform != 'linux'", "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 703c4727c..377565b96 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -106,7 +106,7 @@ candle-transformers = { version = "0.11.1", optional = true } candle-nn = { version = "0.11.1", optional = true } tokenizers = { version = "0.19.1", optional = true } semver = { workspace = true } -roaring = "0.11.4" +roaring = { version = "0.11.4", features = ["serde"] } sha2.workspace = true [dev-dependencies] diff --git a/rust/lancedb/src/table/freshness.rs b/rust/lancedb/src/table/freshness.rs index b2c933cd8..021abe5a9 100644 --- a/rust/lancedb/src/table/freshness.rs +++ b/rust/lancedb/src/table/freshness.rs @@ -4,20 +4,24 @@ //! Source-change detection for computed columns. //! //! A refresh fills nulls, so once a value is durable nothing recomputes it and -//! a later write to one of its inputs leaves it stale forever. Two stamps in -//! the column's field metadata close that: the definition it was computed -//! under, and a per-fragment signature of the input storage it was computed -//! from. A refresh compares both with the manifest and recomputes what -//! disagrees. Signatures are read from manifests, never from data. +//! a later write to one of its inputs leaves it stale forever. The column's +//! field metadata records the definition it was computed under, a +//! per-fragment signature of the input storage it was computed from, and the +//! rows whose retained values are not current for that signature. A refresh +//! compares the stamps with the manifest and recomputes what disagrees. +//! Signatures are read from manifests, never from data. //! //! The map is not stored in the manifest: it is one entry per fragment per //! column, which would dominate the manifest of a large table with many //! computed columns. It lives in an immutable sidecar object under //! `_computed/`, named by its content digest, and the field metadata holds -//! only the reference. Sidecars no retained version references are removed -//! by [`prune_sidecars`]. +//! only the reference. The sidecar also carries row-level exceptions and the +//! accepted repair basis, so old readers reject the newer format and +//! conservatively recompute. Sidecars no retained version references are +//! removed by [`prune_sidecars`]. use std::collections::{BTreeMap, HashMap, HashSet}; +use std::io::{Cursor, Read}; use std::ops::Range; use arrow_array::{Array, UInt64Array}; @@ -33,6 +37,7 @@ use object_store::path::Path; use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use uuid::Uuid; use crate::table::computed_columns::{ DEFINITION_VERSION_META_KEY, RECORDED_AT_VERSION_META_KEY, SOURCE_SIGNATURE_META_KEY, @@ -46,6 +51,110 @@ const MAX_CARRY_FORWARD_VERSIONS: u64 = 1024; /// `{fragment id -> input signature}`. pub type SignatureMap = BTreeMap; +/// Rows whose retained values are not current for the fragment signature. +/// +/// `AllExcept` represents a fragment whose basis changed after a partial +/// refresh: every physical row is dirty except the offsets recorded as clean. +/// It avoids materializing a full bitmap while preserving a partially filled +/// fragment. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DirtyRows { + /// The listed physical offsets are dirty. + Rows(RoaringBitmap), + /// Every physical offset is dirty except the listed clean offsets. + AllExcept(RoaringBitmap), +} + +impl DirtyRows { + fn mark_dirty(&mut self, rows: &RoaringBitmap) { + match self { + Self::Rows(dirty) => *dirty |= rows, + Self::AllExcept(clean) => *clean -= rows, + } + } + + fn mark_clean(&mut self, rows: &RoaringBitmap) { + match self { + Self::Rows(dirty) => *dirty -= rows, + Self::AllExcept(clean) => *clean |= rows, + } + } + + fn is_explicitly_empty(&self) -> bool { + matches!(self, Self::Rows(rows) if rows.is_empty()) + } + + /// Materialize the dirty offsets among `live_rows`. + pub fn live_dirty_rows(&self, live_rows: &RoaringBitmap) -> RoaringBitmap { + match self { + Self::Rows(dirty) => dirty & live_rows, + Self::AllExcept(clean) => live_rows - clean, + } + } +} + +/// `{fragment id -> row-level dirty state}`. A missing fragment is clean. +pub type RowStateMap = BTreeMap; + +/// The single owning value of the freshness sidecar. +/// +/// `SignatureMap` remains the public facade for callers that only need input +/// signatures. The row state and accepted basis live here so a reader can +/// never observe one half of a mixed-generation freshness record. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct FreshnessState { + signatures: SignatureMap, + row_state: RowStateMap, + basis: String, + // A fresh identity for each begin, even when the basis and dirty rows repeat. + intent_id: Uuid, +} + +impl FreshnessState { + fn normalize(mut self) -> Result { + self.row_state.retain(|_, rows| !rows.is_explicitly_empty()); + if self + .row_state + .keys() + .any(|fragment_id| !self.signatures.contains_key(fragment_id)) + { + return Err(invalid( + "row state names a fragment without a signature".to_string(), + )); + } + if self + .signatures + .values() + .any(|signature| !is_signature_hex(signature)) + { + return Err(invalid( + "freshness signature is not a 64-bit hex digest".to_string(), + )); + } + if self.basis.len() > u16::MAX as usize { + return Err(invalid("freshness basis exceeds u16 bytes".to_string())); + } + Ok(self) + } +} + +/// Opaque accepted-basis evidence returned by [`begin_row_repair`] and required by +/// [`finish_row_repair`]. +/// +/// The digest identifies the unified freshness state established before +/// publication, and the selected-row bitmaps bound which rows finalization +/// may change. One accepted wave can finalize disjoint fragments together; +/// later serialized waves preserve clean rows from earlier accepted waves. +/// Each begin persists a fresh intent identity in the sidecar, so repeating +/// the same basis and rows still fences stale non-noop finalization. There is +/// no per-row lease. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RowStateToken { + pub basis: String, + pub intent_digest: String, + pub selected_rows: BTreeMap, +} + /// Every field an input covers, keyed by column path: the field's own id /// first, then its ancestors', because a packed file records the physical /// column under an ancestor's id. @@ -270,12 +379,24 @@ pub enum StoredSignatures { Unreadable, } -/// Directory of signature sidecars, under the dataset root. +/// What the column's stored row-level state says. +#[derive(Debug, PartialEq, Eq)] +pub enum StoredRowStates { + /// No row-level state has been written; every live value is clean. + Absent, + /// Explicit dirty state for the fragments named in the map. + Present(RowStateMap), + /// The owning freshness sidecar cannot be read. + Unreadable, +} + +/// Directory of freshness sidecars, under the dataset root. const SIDECAR_DIR: &str = "_computed"; /// Metadata value prefix referencing a sidecar by its content digest. const SIDECAR_REF: &str = "sidecar:"; const SIDECAR_MAGIC: &[u8; 4] = b"CSIG"; -const SIDECAR_FORMAT: u8 = 1; +const SIDECAR_FORMAT_V1: u8 = 1; +const SIDECAR_FORMAT: u8 = 2; /// How long an unreferenced sidecar is presumed to be a stamp in flight: /// lance's own threshold for unverified files, independent of how much /// version history a cleanup keeps. @@ -293,12 +414,19 @@ fn sidecar_path(dataset: &Dataset, digest: &str) -> Result { .join(format!("{digest}.sig"))) } -/// `CSIG`, format byte, entry count, then one fragment id and 8-byte -/// signature per entry, all little-endian; 12 bytes per fragment. -fn encode_sidecar(map: &SignatureMap) -> Result> { +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn is_signature_hex(value: &str) -> bool { + value.len() == 16 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +#[cfg(test)] +fn encode_legacy_sidecar(map: &SignatureMap) -> Result> { let mut bytes = Vec::with_capacity(9 + map.len() * 12); bytes.extend_from_slice(SIDECAR_MAGIC); - bytes.push(SIDECAR_FORMAT); + bytes.push(SIDECAR_FORMAT_V1); bytes.extend_from_slice( &u32::try_from(map.len()) .map_err(|_| invalid("too many fragments for a signature sidecar".to_string()))? @@ -316,9 +444,55 @@ fn encode_sidecar(map: &SignatureMap) -> Result> { Ok(bytes) } -fn decode_sidecar(bytes: &[u8]) -> Result { +/// `CSIG`, format byte, 16-byte intent UUID, basis, entry count, then for each fragment its id, +/// 8-byte signature, dirty-row mode and serialized bitmap. All integers are +/// little-endian. Format 1 remains readable as clean row state. +fn encode_sidecar(state: &FreshnessState) -> Result> { + let state = state.clone().normalize()?; + let basis_len = u16::try_from(state.basis.len()) + .map_err(|_| invalid("freshness basis exceeds u16 bytes".to_string()))?; + let fragment_count = u32::try_from(state.signatures.len()) + .map_err(|_| invalid("too many fragments for a freshness sidecar".to_string()))?; + let mut bytes = Vec::new(); + bytes.extend_from_slice(SIDECAR_MAGIC); + bytes.push(SIDECAR_FORMAT); + bytes.extend_from_slice(state.intent_id.as_bytes()); + bytes.extend_from_slice(&basis_len.to_le_bytes()); + bytes.extend_from_slice(state.basis.as_bytes()); + bytes.extend_from_slice(&fragment_count.to_le_bytes()); + for (fragment_id, signature) in &state.signatures { + let hash = u64::from_str_radix(signature, 16).map_err(|_| { + invalid(format!( + "signature '{signature}' is not a 64-bit hex digest" + )) + })?; + let rows = state + .row_state + .get(fragment_id) + .cloned() + .unwrap_or_else(|| DirtyRows::Rows(RoaringBitmap::new())); + let (kind, bitmap) = match rows { + DirtyRows::Rows(bitmap) => (0u8, bitmap), + DirtyRows::AllExcept(clean) => (1u8, clean), + }; + let mut encoded = Vec::with_capacity(bitmap.serialized_size()); + bitmap + .serialize_into(&mut encoded) + .map_err(|error| invalid(format!("encoding freshness bitmap: {error}")))?; + let bitmap_len = u32::try_from(encoded.len()) + .map_err(|_| invalid("freshness bitmap exceeds u32 bytes".to_string()))?; + bytes.extend_from_slice(&fragment_id.to_le_bytes()); + bytes.extend_from_slice(&hash.to_le_bytes()); + bytes.push(kind); + bytes.extend_from_slice(&bitmap_len.to_le_bytes()); + bytes.extend_from_slice(&encoded); + } + Ok(bytes) +} + +fn decode_legacy_sidecar(bytes: &[u8]) -> Result { let malformed = || invalid("signature sidecar is malformed".to_string()); - if bytes.len() < 9 || &bytes[..4] != SIDECAR_MAGIC || bytes[4] != SIDECAR_FORMAT { + if bytes.len() < 9 || &bytes[..4] != SIDECAR_MAGIC || bytes[4] != SIDECAR_FORMAT_V1 { return Err(malformed()); } let count = u32::from_le_bytes(bytes[5..9].try_into().map_err(|_| malformed())?) as usize; @@ -326,14 +500,91 @@ fn decode_sidecar(bytes: &[u8]) -> Result { if body.len() != count * 12 { return Err(malformed()); } - Ok(body + let signatures = body .chunks_exact(12) .map(|entry| { let fragment_id = u32::from_le_bytes(entry[..4].try_into().unwrap()); let hash = u64::from_le_bytes(entry[4..].try_into().unwrap()); (fragment_id, format!("{hash:016x}")) }) - .collect()) + .collect(); + Ok(FreshnessState { + signatures, + ..Default::default() + }) +} + +fn decode_sidecar(bytes: &[u8]) -> Result { + if bytes.len() >= 5 && &bytes[..4] == SIDECAR_MAGIC && bytes[4] == SIDECAR_FORMAT_V1 { + return decode_legacy_sidecar(bytes); + } + let malformed = || invalid("freshness sidecar is malformed".to_string()); + let mut cursor = Cursor::new(bytes); + let mut magic = [0u8; 4]; + cursor.read_exact(&mut magic).map_err(|_| malformed())?; + let mut format = [0u8; 1]; + cursor.read_exact(&mut format).map_err(|_| malformed())?; + if &magic != SIDECAR_MAGIC || format[0] != SIDECAR_FORMAT { + return Err(malformed()); + } + let mut intent_id = [0u8; 16]; + cursor.read_exact(&mut intent_id).map_err(|_| malformed())?; + let mut basis_len = [0u8; 2]; + cursor.read_exact(&mut basis_len).map_err(|_| malformed())?; + let mut basis = vec![0u8; u16::from_le_bytes(basis_len) as usize]; + cursor.read_exact(&mut basis).map_err(|_| malformed())?; + let basis = String::from_utf8(basis).map_err(|_| malformed())?; + let mut count = [0u8; 4]; + cursor.read_exact(&mut count).map_err(|_| malformed())?; + let count = u32::from_le_bytes(count) as usize; + let mut state = FreshnessState { + basis, + intent_id: Uuid::from_bytes(intent_id), + ..Default::default() + }; + for _ in 0..count { + let mut fragment_id = [0u8; 4]; + cursor + .read_exact(&mut fragment_id) + .map_err(|_| malformed())?; + let fragment_id = u32::from_le_bytes(fragment_id); + let mut signature = [0u8; 8]; + cursor.read_exact(&mut signature).map_err(|_| malformed())?; + let mut kind = [0u8; 1]; + cursor.read_exact(&mut kind).map_err(|_| malformed())?; + let mut bitmap_len = [0u8; 4]; + cursor + .read_exact(&mut bitmap_len) + .map_err(|_| malformed())?; + let bitmap_len = u32::from_le_bytes(bitmap_len) as usize; + let mut encoded = vec![0u8; bitmap_len]; + cursor.read_exact(&mut encoded).map_err(|_| malformed())?; + let mut bitmap_reader = Cursor::new(encoded.as_slice()); + let bitmap = + RoaringBitmap::deserialize_from(&mut bitmap_reader).map_err(|_| malformed())?; + if bitmap_reader.position() != bitmap_len as u64 { + return Err(malformed()); + } + let dirty = match kind[0] { + 0 => DirtyRows::Rows(bitmap), + 1 => DirtyRows::AllExcept(bitmap), + _ => return Err(malformed()), + }; + if state.signatures.contains_key(&fragment_id) { + return Err(malformed()); + } + state.signatures.insert( + fragment_id, + format!("{:016x}", u64::from_le_bytes(signature)), + ); + if !dirty.is_explicitly_empty() { + state.row_state.insert(fragment_id, dirty); + } + } + if cursor.position() != bytes.len() as u64 { + return Err(malformed()); + } + state.normalize() } fn digest_of(bytes: &[u8]) -> String { @@ -347,11 +598,14 @@ async fn store(dataset: &Dataset) -> Result> { Ok(dataset.object_store(None).await?) } -/// Write `map` as a sidecar and return the metadata value referencing it. -/// The object is named by its digest, so two columns with the same map +/// Write `state` as a sidecar and return the metadata value referencing it. +/// The object is named by its digest, so two columns with the same state /// share one object and a rewrite is idempotent. -async fn write_sidecar(dataset: &Dataset, map: &SignatureMap) -> Result { - let bytes = encode_sidecar(map)?; +async fn write_sidecar(dataset: &Dataset, state: &FreshnessState) -> Result { + write_sidecar_bytes(dataset, encode_sidecar(state)?).await +} + +async fn write_sidecar_bytes(dataset: &Dataset, bytes: Vec) -> Result { let digest = digest_of(&bytes); store(dataset) .await? @@ -360,11 +614,11 @@ async fn write_sidecar(dataset: &Dataset, map: &SignatureMap) -> Result Ok(format!("{SIDECAR_REF}{digest}")) } -/// A map is read from the table's own directory, then through the bases a +/// A sidecar is read from the table's own directory, then through the bases a /// shallow clone reads its data from, where the source stamped it. A copy /// found through a base is put in the table's own directory so it outlives /// the source's pruning; a failed copy only costs the next lookup. -async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { +async fn read_sidecar_bytes(dataset: &Dataset, digest: &str) -> Result { let own = sidecar_path(dataset, digest)?; let store = store(dataset).await?; let bytes = match store.read_one_all(&own).await { @@ -373,7 +627,7 @@ async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { let bytes = read_sidecar_through_bases(dataset, digest).await?; if let Err(error) = store.put(&own, &bytes).await { log::warn!( - "computed column signature sidecar {digest} could not be copied to {own}: {error}" + "computed column freshness sidecar {digest} could not be copied to {own}: {error}" ); } bytes @@ -382,10 +636,14 @@ async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { }; if digest_of(&bytes) != digest { return Err(invalid(format!( - "signature sidecar {digest} does not match its digest" + "freshness sidecar {digest} does not match its digest" ))); } - decode_sidecar(&bytes) + Ok(bytes) +} + +async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { + decode_sidecar(&read_sidecar_bytes(dataset, digest).await?) } async fn read_sidecar_through_bases(dataset: &Dataset, digest: &str) -> Result { @@ -416,11 +674,11 @@ async fn read_sidecar_through_bases(dataset: &Dataset, digest: &str) -> Result) -> Re Ok(()) } -/// Read the column's stored map. An unreadable map is a state, not an error: -/// failing here would make the column permanently unrefreshable, and the -/// unknown recomputes like every other unknown here. -pub async fn stored_signatures(dataset: &Dataset, column: &str) -> StoredSignatures { +enum StoredFreshnessState { + Absent, + Present { + state: FreshnessState, + digest: Option, + }, + Unreadable, +} + +async fn stored_freshness_state(dataset: &Dataset, column: &str) -> StoredFreshnessState { let Some(encoded) = field_meta(dataset, column, SOURCE_SIGNATURE_META_KEY) else { - return StoredSignatures::Absent; + return StoredFreshnessState::Absent; }; // Inline JSON is the declaration's empty seed and the pre-sidecar form. - let read = match encoded.strip_prefix(SIDECAR_REF) { - Some(digest) => read_sidecar(dataset, digest).await, - None => serde_json::from_str(&encoded).map_err(|e| invalid(e.to_string())), + let (read, digest) = match encoded.strip_prefix(SIDECAR_REF) { + Some(digest) => ( + read_sidecar(dataset, digest).await, + Some(digest.to_string()), + ), + None => ( + serde_json::from_str::(&encoded) + .map(|signatures| FreshnessState { + signatures, + ..Default::default() + }) + .map_err(|e| invalid(e.to_string())), + None, + ), }; match read { - Ok(map) => StoredSignatures::Present(map), + Ok(state) => StoredFreshnessState::Present { state, digest }, Err(error) => { log::warn!( - "computed column '{column}' source signature map is unreadable ({error}); every fragment will be recomputed" + "computed column '{column}' freshness state is unreadable ({error}); every fragment will be recomputed" ); - StoredSignatures::Unreadable + StoredFreshnessState::Unreadable } } } +/// Read the column's stored signature map through the versioned freshness +/// state. An unreadable sidecar remains a distinct fail-closed state. +pub async fn stored_signatures(dataset: &Dataset, column: &str) -> StoredSignatures { + match stored_freshness_state(dataset, column).await { + StoredFreshnessState::Absent => StoredSignatures::Absent, + StoredFreshnessState::Present { state, .. } => StoredSignatures::Present(state.signatures), + StoredFreshnessState::Unreadable => StoredSignatures::Unreadable, + } +} + +/// Read the explicit row-level dirty state associated with the column's +/// current signature map. +pub async fn stored_row_states(dataset: &Dataset, column: &str) -> StoredRowStates { + match stored_freshness_state(dataset, column).await { + StoredFreshnessState::Absent => StoredRowStates::Absent, + StoredFreshnessState::Present { state, .. } => StoredRowStates::Present(state.row_state), + StoredFreshnessState::Unreadable => StoredRowStates::Unreadable, + } +} + /// What a refresh must recompute beyond the null rows. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct StalenessPlan { @@ -523,6 +818,8 @@ pub struct StalenessPlan { /// Fragments whose inputs moved since they were computed, or that were /// never recorded. pub dirty: HashSet, + /// Current-signature fragments with only some rows dirty. + pub row_dirty: RowStateMap, /// Fragments a compaction produced from fresh ones, at their current /// signature: not dirty, and for the seal to record. pub inherited: SignatureMap, @@ -532,6 +829,20 @@ impl StalenessPlan { pub fn is_dirty(&self, fragment_id: u32) -> bool { self.recompute_all || self.dirty.contains(&fragment_id) } + + /// Whether any retained row in the fragment is known to be non-current. + pub fn has_dirty_rows(&self, fragment_id: u32) -> bool { + !self.recompute_all + && self + .row_dirty + .get(&fragment_id) + .is_some_and(|rows| !rows.is_explicitly_empty()) + } + + /// Whether the fragment requires any recomputation, whole or row-level. + pub fn needs_recompute(&self, fragment_id: u32) -> bool { + self.is_dirty(fragment_id) || self.has_dirty_rows(fragment_id) + } } /// A version of the log since the stamp that bears on carrying freshness @@ -691,23 +1002,32 @@ async fn holds_values_in( /// compaction since, so a chain of them carries too. Returns the produced /// fragments' signatures at production; the caller compares each with the /// current manifest, which catches anything written to them afterwards. +#[derive(Default)] +struct CarriedForward { + inherited: SignatureMap, + dirty_products: HashSet, +} + async fn carried_forward( dataset: &Dataset, column: &str, stored: &SignatureMap, + row_state: &RowStateMap, inputs: &InputFields, -) -> Result { +) -> Result { let Some(recorded_at) = field_meta(dataset, column, RECORDED_AT_VERSION_META_KEY) .and_then(|version| version.parse::().ok()) else { - return Ok(SignatureMap::new()); + return Ok(CarriedForward::default()); }; let to = dataset.version().version; if to <= recorded_at || to - recorded_at > MAX_CARRY_FORWARD_VERSIONS { - return Ok(SignatureMap::new()); + return Ok(CarriedForward::default()); } let mut fresh = stored.clone(); let mut inherited = SignatureMap::new(); + let mut dirty_products = HashSet::new(); + let mut known_dirty: HashSet = row_state.keys().copied().collect(); let mut appends = Vec::new(); let mut manifests = Manifests { dataset, @@ -723,6 +1043,7 @@ async fn carried_forward( Some(Step::Compaction(compactions)) => compactions, }; for (consumed, produced) in compactions { + let source_dirty = consumed.iter().any(|id| known_dirty.contains(id)); // Each source's signature and live rows just before the // compaction; live rows place its contribution in the product. let mut sources = Vec::with_capacity(consumed.len()); @@ -787,11 +1108,21 @@ async fn carried_forward( if let Some(signature) = signature_of(after, *id, inputs)? { fresh.insert(*id, signature.clone()); inherited.insert(*id, signature); + if source_dirty { + inherited.remove(id); + dirty_products.insert(*id); + known_dirty.insert(*id); + } else { + known_dirty.remove(id); + } } } } } - Ok(inherited) + Ok(CarriedForward { + inherited, + dirty_products, + }) } /// Decide what is stale, from the manifest alone. @@ -809,15 +1140,15 @@ pub async fn staleness_against( definition_version: &str, inputs: &InputFields, ) -> Result { - let stored = match stored_signatures(dataset, column).await { - StoredSignatures::Absent => return Ok(StalenessPlan::default()), - StoredSignatures::Unreadable => { + let stored = match stored_freshness_state(dataset, column).await { + StoredFreshnessState::Absent => return Ok(StalenessPlan::default()), + StoredFreshnessState::Unreadable => { return Ok(StalenessPlan { recompute_all: true, ..Default::default() }); } - StoredSignatures::Present(stored) => stored, + StoredFreshnessState::Present { state: stored, .. } => stored, }; let stored_version = field_meta(dataset, column, DEFINITION_VERSION_META_KEY); if stored_version.is_some_and(|version| version != definition_version) { @@ -826,16 +1157,23 @@ pub async fn staleness_against( ..Default::default() }); } + let FreshnessState { + signatures: stored, + row_state, + .. + } = stored; let unrecorded = dataset .get_fragments() .iter() .any(|fragment| !stored.contains_key(&(fragment.id() as u32))); - let mut inherited = if unrecorded { - carried_forward(dataset, column, &stored, inputs).await? + let carried = if unrecorded { + carried_forward(dataset, column, &stored, &row_state, inputs).await? } else { - SignatureMap::new() + CarriedForward::default() }; - let mut dirty = HashSet::new(); + let mut inherited = carried.inherited; + let mut dirty = carried.dirty_products; + let mut row_dirty = RowStateMap::new(); let mut live = HashSet::new(); let bases = Bases::of(dataset)?; for fragment in dataset.get_fragments() { @@ -844,12 +1182,16 @@ pub async fn staleness_against( let current = fragment_input_signature(&bases, fragment.metadata(), inputs)?; if stored.get(&id).or_else(|| inherited.get(&id)) != Some(¤t) { dirty.insert(id); + } else if let Some(rows) = row_state.get(&id) { + row_dirty.insert(id, rows.clone()); } } inherited.retain(|id, _| live.contains(id) && !dirty.contains(id)); + row_dirty.retain(|id, _| live.contains(id) && !dirty.contains(id)); Ok(StalenessPlan { recompute_all: false, dirty, + row_dirty, inherited, }) } @@ -938,40 +1280,448 @@ pub async fn write_signatures( .iter() .map(|fragment| fragment.id() as u32) .collect(); - // An unreadable map is discarded rather than merged: nothing in it can be - // trusted, and its fragments recompute until a later refresh records them. - let mut merged = match stored_signatures(dataset, column).await { - StoredSignatures::Present(stored) => stored, - StoredSignatures::Absent | StoredSignatures::Unreadable => SignatureMap::new(), + // An unreadable state is discarded rather than merged: nothing in it can + // be trusted, and its fragments recompute until a later refresh records + // them. + let stored = stored_freshness_state(dataset, column).await; + let state_unreadable = matches!(&stored, StoredFreshnessState::Unreadable); + let mut state = match stored { + StoredFreshnessState::Present { state, .. } => state, + StoredFreshnessState::Absent | StoredFreshnessState::Unreadable => { + FreshnessState::default() + } }; - merged.extend(entries); - merged.retain(|fragment_id, _| live.contains(fragment_id)); - // The sidecar is durable before the commit references it; a failure in - // between leaves an unreferenced object for `prune_sidecars`. - let encoded = if merged.is_empty() { - "{}".to_string() + let recomputed: HashSet = entries.keys().copied().collect(); + let signatures_changed = entries + .iter() + .any(|(fragment_id, signature)| state.signatures.get(fragment_id) != Some(signature)); + if state_unreadable + && !live + .iter() + .all(|fragment_id| recomputed.contains(fragment_id)) + { + return Err(invalid( + "cannot replace an unreadable freshness state from a partial signature stamp" + .to_string(), + )); + } + // A signature entry written through this API means the fragment was + // recomputed in full. Whole callers keep their old semantics while + // clearing any partial retry state left by a previous publication. + for fragment_id in &recomputed { + state.row_state.remove(fragment_id); + } + state.signatures.extend(entries); + state + .signatures + .retain(|fragment_id, _| live.contains(fragment_id)); + state + .row_state + .retain(|fragment_id, _| live.contains(fragment_id)); + if state.row_state.is_empty() || state_unreadable || signatures_changed { + state.basis.clear(); + } + Ok( + write_freshness_state(dataset, column, definition_version, state, false, None) + .await? + .version, + ) +} + +struct FreshnessWrite { + version: u64, + intent_digest: Option, +} + +async fn write_freshness_state( + dataset: &mut Dataset, + column: &str, + definition_version: &str, + state: FreshnessState, + retain_empty_sidecar: bool, + expected_latest_version: Option, +) -> Result { + let state = state.normalize()?; + let inline_empty = state.signatures.is_empty() + && state.row_state.is_empty() + && state.basis.is_empty() + && !retain_empty_sidecar; + // The sidecar is durable before the metadata commit references it. + // Unreferenced objects are removed by `prune_sidecars`. + let (signature_ref, intent_digest) = if inline_empty { + ("{}".to_string(), None) } else { - write_sidecar(dataset, &merged).await? + let reference = write_sidecar(dataset, &state).await?; + let digest = reference + .strip_prefix(SIDECAR_REF) + .expect("sidecar reference has its prefix") + .to_string(); + (reference, Some(digest)) }; let describes = dataset.version().version; - dataset - .update_field_metadata() - .update( - column, - [ - (SOURCE_SIGNATURE_META_KEY.to_string(), encoded), - ( - DEFINITION_VERSION_META_KEY.to_string(), - definition_version.to_string(), - ), - ( - RECORDED_AT_VERSION_META_KEY.to_string(), - describes.to_string(), - ), - ], - )? - .await?; - Ok(dataset.version().version) + let updates = vec![ + (SOURCE_SIGNATURE_META_KEY.to_string(), Some(signature_ref)), + ( + DEFINITION_VERSION_META_KEY.to_string(), + Some(definition_version.to_string()), + ), + ( + RECORDED_AT_VERSION_META_KEY.to_string(), + Some(describes.to_string()), + ), + ]; + let builder = dataset.update_field_metadata().update(column, updates)?; + match expected_latest_version { + Some(expected) => builder.with_expected_latest_version(expected).await?, + None => builder.await?, + }; + Ok(FreshnessWrite { + version: dataset.version().version, + intent_digest, + }) +} + +/// One fragment's rows selected by an accepted refresh request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RowRepair { + /// Input signature of the request's pinned source. + pub signature: String, + /// Selected physical offsets. They become dirty before publication. + pub rows: RoaringBitmap, +} + +/// Durably mark every selected row dirty before its data publication. +/// +/// The returned token binds the subsequent filled-row clear to the accepted +/// input basis and selected rows. This function checks the accepted +/// signatures against the latest manifest and does not retry a concurrent +/// metadata conflict with stale state. +/// +/// `basis` identifies the request's accepted input snapshot. It may be reused +/// when retrying the same inputs: each successful begin records a new intent +/// identity independently of that basis. +/// +/// Publication and verification belong to the caller. In this example, +/// `publish_values` writes the selected computed values and retains a receipt; +/// `verify_survival` checks that receipt against the latest snapshot, including +/// any intervening Restore. These two helpers are application-defined. +/// +/// ``` +/// use std::collections::BTreeMap; +/// use lance::Dataset; +/// use lancedb::Result; +/// use lancedb::table::freshness::{ +/// begin_row_repair, finish_row_repair, InputFields, RowRepair, RowStateToken, +/// }; +/// use roaring::RoaringBitmap; +/// # struct Publication { +/// # filled: BTreeMap, +/// # skipped: BTreeMap, +/// # } +/// # async fn publish_values(_: &mut Dataset, _: &RowStateToken) -> Result { +/// # unimplemented!() +/// # } +/// # async fn verify_survival(_: &Dataset, _: &Publication) -> Result<()> { +/// # unimplemented!() +/// # } +/// async fn repair( +/// dataset: &mut Dataset, +/// inputs: &InputFields, +/// selected: &BTreeMap, +/// ) -> Result { +/// let token = begin_row_repair( +/// dataset, "embedding", "definition-v1", inputs, +/// "accepted-input-snapshot", selected, +/// ).await?; +/// let publication = publish_values(dataset, &token).await?; +/// dataset.checkout_latest().await?; +/// verify_survival(dataset, &publication).await?; +/// let verified_version = dataset.version().version; +/// finish_row_repair( +/// dataset, "embedding", "definition-v1", inputs, &token, +/// &publication.filled, &publication.skipped, Some(verified_version), +/// ).await +/// } +/// ``` +/// +/// `verified_version` is the snapshot just checked for publication survival, +/// not the version returned by begin or the original publication. Passing it +/// to [`finish_row_repair`] prevents a later version change from silently +/// rebasing the clean-row update. `None` omits that version fence. +pub async fn begin_row_repair( + dataset: &mut Dataset, + column: &str, + definition_version: &str, + inputs: &InputFields, + basis: impl Into, + repairs: &BTreeMap, +) -> Result { + let basis = basis.into(); + if basis.is_empty() { + return Err(invalid("row repair basis must not be empty".to_string())); + } + if repairs.is_empty() { + return Err(invalid("row repair has no fragments".to_string())); + } + for repair in repairs.values() { + if !is_signature_hex(&repair.signature) { + return Err(invalid( + "row repair signature is not a 64-bit hex digest".to_string(), + )); + } + } + dataset.checkout_latest().await?; + let live: HashSet = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect(); + if let Some(missing) = repairs.keys().find(|id| !live.contains(id)) { + return Err(invalid(format!( + "row repair names fragment {missing}, which is not live" + ))); + } + + let stored_version = field_meta(dataset, column, DEFINITION_VERSION_META_KEY); + let definition_changed = stored_version.is_some_and(|version| version != definition_version); + let stored = stored_freshness_state(dataset, column).await; + let was_absent = matches!(&stored, StoredFreshnessState::Absent); + let state_unreadable = matches!(&stored, StoredFreshnessState::Unreadable); + let mut freshness = match stored { + StoredFreshnessState::Present { state, .. } => state, + StoredFreshnessState::Absent | StoredFreshnessState::Unreadable => { + FreshnessState::default() + } + }; + + if was_absent && !state_unreadable { + freshness.signatures = + signatures_for(dataset, &live.iter().copied().collect::>(), inputs)?; + freshness.row_state.clear(); + freshness.basis.clear(); + } + if state_unreadable || definition_changed { + freshness.signatures = + signatures_for(dataset, &live.iter().copied().collect::>(), inputs)?; + freshness.row_state = live + .iter() + .map(|id| (*id, DirtyRows::AllExcept(RoaringBitmap::new()))) + .collect(); + } + + for (fragment_id, repair) in repairs { + let current = signature_of(dataset, *fragment_id, inputs)?.ok_or_else(|| { + invalid(format!( + "row repair fragment {fragment_id} disappeared from the latest manifest" + )) + })?; + if current != repair.signature { + return Err(invalid(format!( + "row repair fragment {fragment_id} input basis changed before dirty intent" + ))); + } + let preserve = !state_unreadable + && !definition_changed + && freshness.signatures.get(fragment_id) == Some(¤t); + let dirty = if preserve { + freshness + .row_state + .entry(*fragment_id) + .or_insert_with(|| DirtyRows::Rows(RoaringBitmap::new())) + } else { + freshness + .row_state + .insert(*fragment_id, DirtyRows::AllExcept(RoaringBitmap::new())); + freshness + .row_state + .get_mut(fragment_id) + .expect("row state was just inserted") + }; + dirty.mark_dirty(&repair.rows); + freshness.signatures.insert(*fragment_id, current); + } + freshness + .signatures + .retain(|fragment_id, _| live.contains(fragment_id)); + freshness + .row_state + .retain(|fragment_id, _| live.contains(fragment_id)); + freshness.basis = basis.clone(); + freshness.intent_id = Uuid::new_v4(); + + let write = + write_freshness_state(dataset, column, definition_version, freshness, true, None).await?; + let intent_digest = write + .intent_digest + .ok_or_else(|| invalid("row repair dirty intent produced no sidecar".to_string()))?; + Ok(RowStateToken { + basis, + intent_digest, + selected_rows: repairs + .iter() + .map(|(fragment_id, repair)| (*fragment_id, repair.rows.clone())) + .collect(), + }) +} + +fn apply_row_repair( + rows: &mut RowStateMap, + selections: &BTreeMap, + filled_rows: &BTreeMap, + skipped_rows: &BTreeMap, +) -> Result<()> { + if filled_rows + .keys() + .chain(skipped_rows.keys()) + .any(|fragment_id| !selections.contains_key(fragment_id)) + { + return Err(invalid( + "row repair result names a fragment outside its token".to_string(), + )); + } + for (fragment_id, selected) in selections { + if filled_rows + .get(fragment_id) + .zip(skipped_rows.get(fragment_id)) + .is_some_and(|(filled, skipped)| !(filled & skipped).is_empty()) + { + return Err(invalid(format!( + "row repair fragment {fragment_id} marks a row both filled and skipped" + ))); + } + if filled_rows + .get(fragment_id) + .is_some_and(|filled| !filled.is_subset(selected)) + || skipped_rows + .get(fragment_id) + .is_some_and(|skipped| !skipped.is_subset(selected)) + { + return Err(invalid(format!( + "row repair fragment {fragment_id} changes a row outside its selected set" + ))); + } + let state = rows + .entry(*fragment_id) + .or_insert_with(|| DirtyRows::Rows(RoaringBitmap::new())); + if let Some(filled) = filled_rows.get(fragment_id) { + state.mark_clean(filled); + } + if let Some(skipped) = skipped_rows.get(fragment_id) { + state.mark_dirty(skipped); + } + } + rows.retain(|_, state| !state.is_explicitly_empty()); + Ok(()) +} + +/// Clear rows whose publication was accepted and keep every skipped row dirty. +/// +/// The token names the accepted input basis and the exact rows it may change. +/// One accepted wave can finalize disjoint fragments together, and later +/// serialized waves preserve clean rows from earlier accepted waves. A newer +/// dirty intent fences a stale non-noop finalization; exact replay of the +/// accepted transition is idempotent. This API deliberately does not invent a +/// second per-row lease. +/// +/// `verified_version` is the latest version whose publication survival the +/// caller verified. With `Some`, finalization commits field metadata +/// conditionally from that exact version, so a Restore before or during the +/// commit fails instead of being rebased underneath the verified publication; +/// with `None`, it uses ordinary metadata-commit semantics and provides no +/// version fence. +#[allow(clippy::too_many_arguments)] +pub async fn finish_row_repair( + dataset: &mut Dataset, + column: &str, + definition_version: &str, + inputs: &InputFields, + token: &RowStateToken, + filled_rows: &BTreeMap, + skipped_rows: &BTreeMap, + verified_version: Option, +) -> Result { + if !is_sha256_hex(&token.intent_digest) { + return Err(invalid("row repair token is malformed".to_string())); + } + if filled_rows + .keys() + .chain(skipped_rows.keys()) + .any(|fragment_id| !token.selected_rows.contains_key(fragment_id)) + { + return Err(invalid( + "row repair result names a fragment outside its token".to_string(), + )); + } + dataset.checkout_latest().await?; + if verified_version.is_some_and(|version| dataset.version().version != version) { + return Err(invalid( + "row repair latest version changed after publication verification".to_string(), + )); + } + let (mut state, current_digest) = match stored_freshness_state(dataset, column).await { + StoredFreshnessState::Present { + state, + digest: Some(digest), + } => (state, digest), + StoredFreshnessState::Present { digest: None, .. } => { + return Err(invalid( + "row repair token cannot name an inline freshness state".to_string(), + )); + } + StoredFreshnessState::Absent | StoredFreshnessState::Unreadable => { + return Err(invalid( + "row repair freshness state disappeared before finalization".to_string(), + )); + } + }; + if state.basis != token.basis { + return Err(invalid( + "row repair token does not match the current dirty intent".to_string(), + )); + } + for fragment_id in token.selected_rows.keys() { + let expected = state.signatures.get(fragment_id).ok_or_else(|| { + invalid(format!( + "row repair fragment {fragment_id} lost its signature" + )) + })?; + let current = signature_of(dataset, *fragment_id, inputs)?.ok_or_else(|| { + invalid(format!( + "row repair fragment {fragment_id} disappeared before finalization" + )) + })?; + if expected != ¤t { + return Err(invalid(format!( + "row repair fragment {fragment_id} input basis changed before finalization" + ))); + } + } + let before = state.row_state.clone(); + apply_row_repair( + &mut state.row_state, + &token.selected_rows, + filled_rows, + skipped_rows, + )?; + if state.row_state == before { + return Ok(dataset.version().version); + } + if current_digest != token.intent_digest.as_str() { + return Err(invalid( + "row repair dirty intent changed before finalization".to_string(), + )); + } + state.basis = token.basis.clone(); + let write = write_freshness_state( + dataset, + column, + definition_version, + state, + true, + verified_version, + ) + .await?; + Ok(write.version) } #[cfg(test)] @@ -1063,6 +1813,737 @@ mod tests { .unwrap(); } + fn fragment_signature(dataset: &Dataset, fragment_id: u32) -> String { + signature_of(dataset, fragment_id, &inputs(dataset)) + .unwrap() + .unwrap() + } + + fn repair(dataset: &Dataset, fragment_id: u32, rows: RoaringBitmap) -> RowRepair { + RowRepair { + signature: fragment_signature(dataset, fragment_id), + rows, + } + } + + #[test] + fn unified_sidecar_round_trips_partial_clean_exceptions() { + let state = FreshnessState { + signatures: SignatureMap::from([ + (3, "0000000000000003".to_string()), + (7, "0000000000000007".to_string()), + ]), + basis: "job-generation".to_string(), + intent_id: Uuid::from_u128(7), + row_state: RowStateMap::from([ + (3, DirtyRows::Rows(RoaringBitmap::from_iter([1, 4, 9]))), + (7, DirtyRows::AllExcept(RoaringBitmap::from_iter([0, 2]))), + ]), + }; + let encoded = encode_sidecar(&state).unwrap(); + assert_eq!(encoded[4], SIDECAR_FORMAT); + assert!( + decode_legacy_sidecar(&encoded).is_err(), + "an old reader must reject the row-aware format" + ); + assert_eq!(decode_sidecar(&encoded).unwrap(), state); + } + + #[test] + fn legacy_sidecar_is_read_as_clean_rows() { + let signatures = SignatureMap::from([ + (1, "0000000000000001".to_string()), + (2, "0000000000000002".to_string()), + ]); + let state = decode_sidecar(&encode_legacy_sidecar(&signatures).unwrap()).unwrap(); + assert_eq!(state.signatures, signatures); + assert!(state.row_state.is_empty()); + assert!(state.basis.is_empty()); + } + + #[tokio::test] + async fn a_row_repair_clears_filled_rows_and_retains_skipped_rows() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1, 2])))]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-job-generation-a", + &repairs, + ) + .await + .unwrap(); + + let before = plan(&dataset).await; + assert!(before.dirty.is_empty()); + assert_eq!( + before.row_dirty.get(&0), + Some(&DirtyRows::Rows(RoaringBitmap::from_iter([1, 2]))) + ); + + let version = finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + &BTreeMap::from([(0, RoaringBitmap::from_iter([2]))]), + None, + ) + .await + .unwrap(); + assert_eq!( + stored_row_states(&dataset, COLUMN).await, + StoredRowStates::Present(RowStateMap::from([( + 0, + DirtyRows::Rows(RoaringBitmap::from_iter([2])), + )])) + ); + assert_eq!( + plan(&dataset).await.row_dirty.get(&0), + Some(&DirtyRows::Rows(RoaringBitmap::from_iter([2]))) + ); + + // Replaying the accepted transition changes nothing. + assert_eq!( + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + &BTreeMap::from([(0, RoaringBitmap::from_iter([2]))]), + None, + ) + .await + .unwrap(), + version + ); + } + + #[tokio::test] + async fn a_definition_change_preserves_filled_clean_exceptions() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let ids = inputs(&dataset); + let definition = "definition-generation-b"; + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([0])))]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + definition, + &ids, + "accepted-job-generation-b", + &repairs, + ) + .await + .unwrap(); + finish_row_repair( + &mut dataset, + COLUMN, + definition, + &ids, + &token, + &BTreeMap::from([(0, RoaringBitmap::from_iter([0]))]), + &BTreeMap::new(), + None, + ) + .await + .unwrap(); + + let plan = staleness_against(&dataset, COLUMN, definition, &ids) + .await + .unwrap(); + assert!(plan.dirty.is_empty()); + assert_eq!( + plan.row_dirty.get(&0), + Some(&DirtyRows::AllExcept(RoaringBitmap::from_iter([0]))) + ); + assert_eq!( + plan.row_dirty.get(&1), + Some(&DirtyRows::AllExcept(RoaringBitmap::new())) + ); + } + + #[tokio::test] + async fn one_token_finishes_disjoint_fragments_without_clearing_other_rows() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([ + (0, repair(&dataset, 0, RoaringBitmap::from_iter([1]))), + (1, repair(&dataset, 1, RoaringBitmap::from_iter([2]))), + ]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + &BTreeMap::from([(1, RoaringBitmap::from_iter([2]))]), + None, + ) + .await + .unwrap(); + assert_eq!( + stored_row_states(&dataset, COLUMN).await, + StoredRowStates::Present(RowStateMap::from([( + 1, + DirtyRows::Rows(RoaringBitmap::from_iter([2])), + )])) + ); + } + + #[tokio::test] + async fn a_new_intent_blocks_a_stale_non_noop_finish() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let first_repair = + BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let first = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &first_repair, + ) + .await + .unwrap(); + let second_repair = + BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([3])))]); + begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &second_repair, + ) + .await + .unwrap(); + let before = stored_row_states(&dataset, COLUMN).await; + let error = finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &first, + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + &BTreeMap::new(), + None, + ) + .await + .expect_err("a newer dirty intent must fence a non-noop finish"); + assert!( + error.to_string().contains("dirty intent changed"), + "{error}" + ); + assert_eq!(stored_row_states(&dataset, COLUMN).await, before); + } + + #[tokio::test] + async fn repeated_row_repair_intent_fences_old_completion() { + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + let mut dataset = table(uri).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let selected = BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]); + let first = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &first, + &selected, + &BTreeMap::new(), + None, + ) + .await + .unwrap(); + + let second = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + let verified = dataset.version().version; + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &second, + &BTreeMap::new(), + &selected, + Some(verified), + ) + .await + .unwrap(); + + let expected = StoredRowStates::Present(RowStateMap::from([( + 0, + DirtyRows::Rows(RoaringBitmap::from_iter([1])), + )])); + let mut reopened = Dataset::open(uri).await.unwrap(); + assert_eq!(stored_row_states(&reopened, COLUMN).await, expected); + let verified = reopened.version().version; + let error = finish_row_repair( + &mut reopened, + COLUMN, + &definition, + &ids, + &first, + &selected, + &BTreeMap::new(), + Some(verified), + ) + .await + .expect_err("an old completion must not clear a newer skipped repair of the same rows"); + assert!( + error.to_string().contains("dirty intent changed"), + "{error}" + ); + assert_eq!(reopened.version().version, verified); + let reopened = Dataset::open(uri).await.unwrap(); + assert_eq!(stored_row_states(&reopened, COLUMN).await, expected); + } + + #[tokio::test] + async fn recovery_rejects_a_restore_after_publication_verification() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + let intent_version = dataset.version().version; + let mut publication = dataset.clone(); + publication + .update_field_metadata() + .update(COLUMN, [("fixture.publication", "1")]) + .unwrap() + .await + .unwrap(); + let verified = publication.version().version; + + let mut intent = dataset.checkout_version(intent_version).await.unwrap(); + intent.restore().await.unwrap(); + let before = stored_row_states(&dataset, COLUMN).await; + let error = finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + &BTreeMap::new(), + Some(verified), + ) + .await + .expect_err("a restore after verification must fence recovery finalization"); + + assert!( + error + .to_string() + .contains("latest version changed after publication verification"), + "{error}" + ); + assert_eq!(stored_row_states(&dataset, COLUMN).await, before); + assert_eq!( + plan(&dataset).await.row_dirty.get(&0), + Some(&DirtyRows::Rows(RoaringBitmap::from_iter([1]))), + "the recovered row must remain retryable" + ); + } + + #[tokio::test] + async fn an_all_success_finish_replays_without_a_legacy_sidecar() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + let filled = BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]); + let version = finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &filled, + &BTreeMap::new(), + None, + ) + .await + .unwrap(); + assert_eq!( + stored_row_states(&dataset, COLUMN).await, + StoredRowStates::Present(RowStateMap::new()), + "an accepted all-success transition keeps an empty freshness sidecar for replay" + ); + assert_eq!( + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &filled, + &BTreeMap::new(), + None, + ) + .await + .unwrap(), + version + ); + + corrupt_current_sidecar(&dataset).await; + let error = finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &filled, + &BTreeMap::new(), + None, + ) + .await + .expect_err("legacy absence cannot prove an accepted generation"); + assert!( + error + .to_string() + .contains("disappeared before finalization"), + "{error}" + ); + } + + #[tokio::test] + async fn finish_rejects_rows_outside_the_token_selection() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + let before = stored_row_states(&dataset, COLUMN).await; + let error = finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &BTreeMap::from([(0, RoaringBitmap::from_iter([2]))]), + &BTreeMap::new(), + None, + ) + .await + .expect_err("a token must not clear rows it did not select"); + assert!( + error + .to_string() + .contains("changes a row outside its selected set"), + "{error}" + ); + assert_eq!(stored_row_states(&dataset, COLUMN).await, before); + } + + #[tokio::test] + async fn unreadable_row_state_keeps_unselected_fragments_dirty() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let stale = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &stale, + &BTreeMap::new(), + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + None, + ) + .await + .unwrap(); + corrupt_current_sidecar(&dataset).await; + + let retry = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([2])))]); + begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "retry-input-basis", + &retry, + ) + .await + .unwrap(); + let plan = plan(&dataset).await; + assert_eq!( + plan.row_dirty.get(&0), + Some(&DirtyRows::AllExcept(RoaringBitmap::new())) + ); + assert_eq!( + plan.row_dirty.get(&1), + Some(&DirtyRows::AllExcept(RoaringBitmap::new())), + "unreadable state outside the selected fragment must stay dirty" + ); + } + + #[tokio::test] + async fn write_signatures_fails_closed_for_unreadable_row_state() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let stale = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &stale, + &BTreeMap::new(), + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + None, + ) + .await + .unwrap(); + corrupt_current_sidecar(&dataset).await; + + let partial = SignatureMap::from([(0, fragment_signature(&dataset, 0))]); + let error = write_signatures(&mut dataset, COLUMN, &definition, partial) + .await + .expect_err("a partial stamp cannot replace unreadable state"); + assert!( + error.to_string().contains("partial signature stamp"), + "{error}" + ); + assert!(plan(&dataset).await.recompute_all); + } + + #[tokio::test] + async fn concurrent_field_metadata_writes_on_one_field_conflict() { + let dir = tempfile::tempdir().unwrap(); + let dataset = table(dir.path().to_str().unwrap()).await; + let mut left = dataset.clone(); + let mut right = dataset.clone(); + let left_write = async { + left.update_field_metadata() + .update(COLUMN, [("computed_refresh.concurrent_a", "1")]) + .unwrap() + .await + }; + let right_write = async { + right + .update_field_metadata() + .update(COLUMN, [("computed_refresh.concurrent_b", "2")]) + .unwrap() + .await + }; + let (left_result, right_result) = tokio::join!(left_write, right_write); + assert_eq!( + usize::from(left_result.is_ok()) + usize::from(right_result.is_ok()), + 1, + "same-field metadata writes must conflict: {left_result:?} {right_result:?}" + ); + } + + #[tokio::test] + async fn pruning_tracks_unified_freshness_sidecars() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-input-basis", + &repairs, + ) + .await + .unwrap(); + assert_eq!(sidecar_names(&dataset).await.len(), 2); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 0); + + dataset + .update_field_metadata() + .update( + COLUMN, + [(SOURCE_SIGNATURE_META_KEY.to_string(), "{}".to_string())], + ) + .unwrap() + .await + .unwrap(); + dataset + .cleanup_old_versions(chrono::Duration::zero(), Some(true), None) + .await + .unwrap(); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 2); + assert!(sidecar_names(&dataset).await.is_empty()); + } + + #[tokio::test] + async fn corrupt_unified_freshness_state_fails_closed() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-job-generation-a", + &repairs, + ) + .await + .unwrap(); + corrupt_current_sidecar(&dataset).await; + assert!(plan(&dataset).await.recompute_all); + } + + #[tokio::test] + async fn compaction_from_a_partially_dirty_source_recomputes_the_product() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let definition = definition_version(EXPRESSION); + let ids = inputs(&dataset); + let repairs = BTreeMap::from([(0, repair(&dataset, 0, RoaringBitmap::from_iter([1])))]); + let token = begin_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + "accepted-job-generation-a", + &repairs, + ) + .await + .unwrap(); + finish_row_repair( + &mut dataset, + COLUMN, + &definition, + &ids, + &token, + &BTreeMap::new(), + &BTreeMap::from([(0, RoaringBitmap::from_iter([1]))]), + None, + ) + .await + .unwrap(); + let product = compact(&mut dataset).await; + let plan = plan(&dataset).await; + assert!(plan.dirty.contains(&product), "{plan:?}"); + assert!(!plan.row_dirty.contains_key(&product), "{plan:?}"); + } + /// Strip the refresh's own keys: a column from before signatures existed. async fn make_legacy(dataset: &mut Dataset) { let declaration = dataset @@ -1255,6 +2736,19 @@ mod tests { field_meta(dataset, column, SOURCE_SIGNATURE_META_KEY).unwrap() } + async fn corrupt_current_sidecar(dataset: &Dataset) { + let digest = signature_ref(dataset, COLUMN) + .strip_prefix(SIDECAR_REF) + .unwrap() + .to_string(); + store(dataset) + .await + .unwrap() + .put(&sidecar_path(dataset, &digest).unwrap(), b"CSIG garbage") + .await + .unwrap(); + } + /// The manifest carries only a digest; the map itself is a sidecar the /// reader fetches and verifies. The declaration's empty seed stays inline. #[tokio::test] @@ -1399,7 +2893,15 @@ mod tests { let mut dataset = table(dir.path().to_str().unwrap()).await; let ids = inputs(&dataset); let entries = signatures_for(&dataset, &[0, 1], &ids).unwrap(); - let reference = write_sidecar(&dataset, &entries).await.unwrap(); + let reference = write_sidecar( + &dataset, + &FreshnessState { + signatures: entries, + ..Default::default() + }, + ) + .await + .unwrap(); let removed = prune_sidecars(&dataset, false).await.unwrap(); assert_eq!(removed, 0); @@ -1414,9 +2916,15 @@ mod tests { StoredSignatures::Present(_) )); - let orphan = write_sidecar(&dataset, &SignatureMap::from([(9, "0".repeat(16))])) - .await - .unwrap(); + let orphan = write_sidecar( + &dataset, + &FreshnessState { + signatures: SignatureMap::from([(9, "0".repeat(16))]), + ..Default::default() + }, + ) + .await + .unwrap(); assert_eq!( prune_sidecars(&dataset, false).await.unwrap(), 0, diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index 2d26e1c02..1f0b80829 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -2190,7 +2190,7 @@ mod lsm_tests { } fn insert_batches( &self, - batches: &[lance::dataset::mem_wal::write::StoredBatch], + batches: &[lance::dataset::mem_wal::memtable::batch_store::StoredBatch], ) -> lance_core::Result<()> { self.0.insert_batches(batches) } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index ab55af50c..330cdfb6d 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -192,7 +192,9 @@ async fn execute_refresh_column_with_source( // A recompute rewrites every live row, so it is staged without the // probe and counted as it fills; a null fill probes first, since a // fragment with nothing to gain is not worth a write. - let recompute = staleness.is_dirty(fragment_id); + // Expression refreshes replace a whole column file, so any row-level + // dirty state makes this fragment a whole-fragment recompute here. + let recompute = staleness.needs_recompute(fragment_id); let whole = recompute || { let (gained, unfilled) = count_fragment_gains(&dataset, &fragment, &bound, column).await?;