Compare commits

...

4 Commits

Author SHA1 Message Date
Brendan Clement cb6a1fafa6 refactor(remote): drop x-lancedb-min-version; use timestamp baseline for read-your-write 2026-05-28 20:05:10 -07:00
Will Jones a3339b7bdd ci: drop manylinux2_17 wheel builds (#3455)
manylinux2_17 reached EOL in 2024 and pyarrow stopped publishing 2_17
wheels long ago. We already build manylinux2_28 wheels, so drop the 2_17
matrix entries.

Fixes #3452

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-28 11:30:42 -07:00
Will Jones b20cdc4f93 ci: fix pypi publish on mac/windows/arm (#3449)
The python-v0.32.0 publish run failed on every build matrix entry. Three
independent issues:

1. **Mac and Windows**: `pypa/gh-action-pypi-publish` only runs on
Linux, but was being called inline from each build job.
2. **Linux (all arches)**: `pypa/gh-action-pypi-publish` derives its
docker image name from `github.action_repository`, which is empty when
the action is invoked from inside a composite action
(actions/runner#2473 — pypa's own `action.yml` references this bug). It
falls back to `github.repository`, generating
`docker://ghcr.io/lancedb/lancedb:<tag>`, which doesn't exist →
`denied`. Only the ARM matrix entry surfaced this because it failed
first and cancel-cascaded the rest.
3. **Windows**: `upload-artifact` in `build_windows_wheel` pointed at
`python\target\wheels`, but maturin writes to the workspace-root
`target/wheels`. The artifact was always empty. Also, `pypi-publish.yml`
passed a `vcpkg_token` input that the composite doesn't declare.

## Changes

- Build jobs (linux/mac/windows) now upload their wheels as
`actions/upload-artifact` artifacts.
- New Linux `publish` job downloads all wheel artifacts and runs the
Fury or PyPA publish step directly (not via a composite), so
`github.action_repository` resolves correctly.
- Delete the unused `upload_wheel` composite action.
- Drop the broken upload-artifact step inside `build_windows_wheel`.
- Remove the bogus `vcpkg_token` input.
- Fury upload now loops over all wheels instead of just the first.
- Bump `actions/checkout`, `actions/upload-artifact`,
`actions/download-artifact` to current major versions (Node 24) to clear
deprecation warnings.
- Bump Windows job timeout 60 → 90 minutes; previous run was
cancel-timing-out on a 60m cap.
- Use `rust-lld` as the Windows MSVC linker via
`CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER`. `link.exe` is
single-threaded and the long pole on Windows builds.

Fixes #3445

## Test plan

- [x] Open this PR — `paths` filter triggers a dry-run build on all
three platforms.
- [x] Verify all three builds produce wheels.
- [x] Confirm the `pypa/gh-action-pypi-publish` container actually
starts (the actions/runner#2473 bug) via the `publish-dry-run` job
pointed at TestPyPI.
- [x] **REMOVE BEFORE MERGE**: drop the `publish-dry-run` job and the
now-redundant `actions/upload-artifact` runs on PRs (currently always-on
so the dry-run has wheels to publish).
- [ ] After merge, cherry-pick onto `python-v0.32.0` and force-push the
tag to re-trigger the publish.
2026-05-27 13:43:42 -07:00
LanceDB Robot e77a62e35a chore: update lance dependency to v7.1.0-beta.4 (#3450)
## Summary

- Updates Lance Rust workspace dependencies to `v7.1.0-beta.4` using
`ci/set_lance_version.py`.
- Updates the Java `lance-core` dependency property to `7.1.0-beta.4`.
- Triggering Lance tag:
https://github.com/lance-format/lance/releases/tag/v7.1.0-beta.4

## Verification

- `cargo clippy --workspace --tests --all-features -- -D warnings`
- `cargo fmt --all`

Co-authored-by: Daniel Rammer <hamersaw@protonmail.com>
2026-05-27 08:45:28 -05:00
7 changed files with 231 additions and 219 deletions
@@ -29,7 +29,3 @@ runs:
args: ${{ inputs.args }}
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
working-directory: python
- uses: actions/upload-artifact@v4
with:
name: windows-wheels
path: python\target\wheels
+78 -32
View File
@@ -8,6 +8,9 @@ on:
# This should trigger a dry run (we skip the final publish step)
paths:
- .github/workflows/pypi-publish.yml
- .github/workflows/build_linux_wheel/action.yml
- .github/workflows/build_mac_wheel/action.yml
- .github/workflows/build_windows_wheel/action.yml
- Cargo.toml # Change in dependency frequently breaks builds
- Cargo.lock
@@ -21,32 +24,21 @@ jobs:
linux:
name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
timeout-minutes: 60
permissions:
id-token: write
contents: read
strategy:
matrix:
config:
- platform: x86_64
manylinux: "2_17"
extra_args: ""
runner: ubuntu-22.04
- platform: x86_64
manylinux: "2_28"
extra_args: "--features fp16kernels"
runner: ubuntu-22.04
- platform: aarch64
manylinux: "2_17"
extra_args: ""
# For successful fat LTO builds, we need a large runner to avoid OOM errors.
runner: ubuntu-2404-8x-arm64
# For successful fat LTO builds, we need a large runner to avoid OOM errors.
- platform: aarch64
manylinux: "2_28"
extra_args: "--features fp16kernels"
runner: ubuntu-2404-8x-arm64
runs-on: ${{ matrix.config.runner }}
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
@@ -60,15 +52,14 @@ jobs:
args: "--release --strip ${{ matrix.config.extra_args }}"
arm-build: ${{ matrix.config.platform == 'aarch64' }}
manylinux: ${{ matrix.config.manylinux }}
- uses: ./.github/workflows/upload_wheel
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/python-v')
with:
fury_token: ${{ secrets.FURY_TOKEN }}
name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
path: target/wheels/lancedb-*.whl
if-no-files-found: error
mac:
timeout-minutes: 90
permissions:
id-token: write
contents: read
runs-on: ${{ matrix.config.runner }}
strategy:
matrix:
@@ -78,7 +69,7 @@ jobs:
env:
MACOSX_DEPLOYMENT_TARGET: 10.15
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
@@ -90,18 +81,21 @@ jobs:
with:
python-minor-version: 10
args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
- uses: ./.github/workflows/upload_wheel
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/python-v')
with:
fury_token: ${{ secrets.FURY_TOKEN }}
name: wheels-mac-${{ matrix.config.target }}
path: target/wheels/lancedb-*.whl
if-no-files-found: error
windows:
timeout-minutes: 60
permissions:
id-token: write
contents: read
timeout-minutes: 90
runs-on: windows-latest
env:
# link.exe is single-threaded and the long pole on Windows builds. Use
# rustc's bundled lld-link instead.
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
@@ -113,18 +107,70 @@ jobs:
with:
python-minor-version: 10
args: "--release --strip"
vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
- uses: ./.github/workflows/upload_wheel
- uses: actions/upload-artifact@v7
if: startsWith(github.ref, 'refs/tags/python-v')
with:
fury_token: ${{ secrets.FURY_TOKEN }}
name: wheels-windows
path: target/wheels/lancedb-*.whl
if-no-files-found: error
publish:
name: Publish wheels
if: startsWith(github.ref, 'refs/tags/python-v')
needs: [linux, mac, windows]
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
steps:
- uses: actions/checkout@v6
- name: Download wheel artifacts
uses: actions/download-artifact@v8
with:
pattern: wheels-*
path: target/wheels
merge-multiple: true
- name: List wheels
run: ls -la target/wheels
- name: Choose repo
id: choose_repo
run: |
if [[ ${{ github.ref }} == *beta* ]]; then
echo "repo=fury" >> $GITHUB_OUTPUT
else
echo "repo=pypi" >> $GITHUB_OUTPUT
fi
- name: Publish to Fury
if: steps.choose_repo.outputs.repo == 'fury'
env:
FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
run: |
shopt -s nullglob
WHEELS=(target/wheels/lancedb-*.whl)
if [[ ${#WHEELS[@]} -eq 0 ]]; then
echo "No wheels found in target/wheels/" >&2
exit 1
fi
for WHEEL in "${WHEELS[@]}"; do
echo "Uploading $WHEEL to Fury"
curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
done
# NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
# workflow file, not from inside a composite action. When called from a
# composite, `github.action_repository` is empty (actions/runner#2473)
# and the action falls back to `github.repository`, producing a bogus
# `docker://ghcr.io/<repo>:<ref>` image reference that GHA tries to pull.
- name: Publish to PyPI
if: steps.choose_repo.outputs.repo == 'pypi'
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: target/wheels/
gh-release:
if: startsWith(github.ref, 'refs/tags/python-v')
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
with:
fetch-depth: 0
lfs: true
@@ -187,13 +233,13 @@ jobs:
report-failure:
name: Report Workflow Failure
runs-on: ubuntu-latest
needs: [linux, mac, windows]
needs: [linux, mac, windows, publish]
permissions:
contents: read
issues: write
if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@v6
- uses: ./.github/actions/create-failure-issue
with:
job-results: ${{ toJSON(needs) }}
-34
View File
@@ -1,34 +0,0 @@
name: upload-wheel
description: "Upload wheels to Pypi"
inputs:
fury_token:
required: true
description: "release token for the fury repo"
runs:
using: "composite"
steps:
- name: Choose repo
shell: bash
id: choose_repo
run: |
if [[ ${{ github.ref }} == *beta* ]]; then
echo "repo=fury" >> $GITHUB_OUTPUT
else
echo "repo=pypi" >> $GITHUB_OUTPUT
fi
- name: Publish to Fury
if: steps.choose_repo.outputs.repo == 'fury'
shell: bash
env:
FURY_TOKEN: ${{ inputs.fury_token }}
run: |
WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
echo "Uploading $WHEEL to Fury"
curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
- name: Publish to PyPI
if: steps.choose_repo.outputs.repo == 'pypi'
uses: pypa/gh-action-pypi-publish@release/v1
with:
packages-dir: target/wheels/
Generated
+60 -54
View File
@@ -976,16 +976,15 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
version = "1.11.3"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182"
checksum = "0504b1ab12debb5959e5165ee5fe97dd387e7aa7ea6a477bfd7635dfe769a4f5"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http-client",
"aws-smithy-observability",
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
"bytes",
"fastrand",
@@ -1002,9 +1001,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
version = "1.12.1"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc117c179ecf39a62a0a3f49f600e9ac26a7ad7dd172177999f83933af776c32"
checksum = "b71a13df6ada0aafbf21a73bdfcdf9324cfa9df77d96b8446045be3cde61b42e"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api-macros",
@@ -1029,22 +1028,11 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "aws-smithy-schema"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 1.4.0",
]
[[package]]
name = "aws-smithy-types"
version = "1.4.8"
version = "1.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056b66dbce2f81cc0c1e2b05bb402eb58f8a3530479d650efadd5bbae9a4050b"
checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
dependencies = [
"base64-simd",
"bytes",
@@ -3296,8 +3284,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-array",
"rand 0.9.4",
@@ -4518,8 +4506,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arc-swap",
"arrow",
@@ -4564,6 +4552,7 @@ dependencies = [
"lance-io",
"lance-linalg",
"lance-namespace",
"lance-select",
"lance-table",
"lance-tokenizer",
"log",
@@ -4591,8 +4580,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4612,8 +4601,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrayref",
"paste",
@@ -4622,8 +4611,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4658,8 +4647,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow",
"arrow-array",
@@ -4689,8 +4678,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow",
"arrow-array",
@@ -4708,8 +4697,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4744,8 +4733,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -4776,8 +4765,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arc-swap",
"arrow",
@@ -4814,6 +4803,7 @@ dependencies = [
"lance-file",
"lance-io",
"lance-linalg",
"lance-select",
"lance-table",
"lance-tokenizer",
"libm",
@@ -4841,8 +4831,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow",
"arrow-arith",
@@ -4884,8 +4874,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4901,8 +4891,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow",
"async-trait",
@@ -4914,8 +4904,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow",
"arrow-ipc",
@@ -4962,10 +4952,25 @@ dependencies = [
"url",
]
[[package]]
name = "lance-select"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-array",
"arrow-buffer",
"byteorder",
"bytes",
"deepsize",
"itertools 0.13.0",
"lance-core",
"roaring",
]
[[package]]
name = "lance-table"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow",
"arrow-array",
@@ -4984,6 +4989,7 @@ dependencies = [
"lance-core",
"lance-file",
"lance-io",
"lance-select",
"log",
"object_store",
"prost",
@@ -5004,8 +5010,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5016,8 +5022,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "7.0.0-rc.1"
source = "git+https://github.com/lance-format/lance.git?tag=v7.0.0-rc.1#06f52e901dd9bd99d299263130fb2bda0a3f91af"
version = "7.1.0-beta.4"
source = "git+https://github.com/lance-format/lance.git?tag=v7.1.0-beta.4#0c0b3e18c0a4c75bda1dd6ec9d6247ef75bd29d9"
dependencies = [
"jieba-rs",
"lindera",
@@ -8315,9 +8321,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.150"
version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=7.0.0-rc.1", default-features = false, "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=7.0.0-rc.1", default-features = false, "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=7.0.0-rc.1", default-features = false, "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=7.0.0-rc.1", "tag" = "v7.0.0-rc.1", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=7.1.0-beta.4", default-features = false, "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=7.1.0-beta.4", default-features = false, "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=7.1.0-beta.4", default-features = false, "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=7.1.0-beta.4", "tag" = "v7.1.0-beta.4", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>7.0.0-beta.13</lance-core.version>
<lance-core.version>7.1.0-beta.4</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+78 -80
View File
@@ -66,47 +66,36 @@ use std::time::{Duration, SystemTime};
use tokio::sync::RwLock;
const REQUEST_TIMEOUT_HEADER: HeaderName = HeaderName::from_static("x-request-timeout-ms");
const MIN_VERSION_HEADER: HeaderName = HeaderName::from_static("x-lancedb-min-version");
const MIN_TIMESTAMP_HEADER: HeaderName = HeaderName::from_static("x-lancedb-min-timestamp");
const METRIC_TYPE_KEY: &str = "metric_type";
const INDEX_TYPE_KEY: &str = "index_type";
const SCHEMA_CACHE_TTL: Duration = Duration::from_secs(30);
const SCHEMA_CACHE_REFRESH_WINDOW: Duration = Duration::from_secs(5);
/// Per-table state driving the freshness headers (`x-lancedb-min-version` and
/// `x-lancedb-min-timestamp`) sent on read requests.
/// Per-table state driving the `x-lancedb-min-timestamp` freshness header
/// sent on read requests.
#[derive(Debug, Default, Clone, Copy)]
struct FreshnessState {
/// Provides read-your-write within a single handle: writes that return a
/// version update this, and reads send it as `x-lancedb-min-version`.
min_version: Option<u64>,
/// Wall-clock time captured at the last [`BaseTable::checkout_latest`]
/// call. Subsequent reads send
/// `max(baseline, now - read_consistency_interval)` as
/// `x-lancedb-min-timestamp`.
/// Wall-clock floor for read freshness, bumped to `now` whenever this
/// handle performs a write or an explicit [`BaseTable::checkout_latest`].
/// Subsequent reads send `max(baseline, now - read_consistency_interval)`
/// as `x-lancedb-min-timestamp`.
///
/// Without this, `checkout_latest()` would have no effect on subsequent
/// reads when `read_consistency_interval` is unset (the default): a
/// server-side cache could still serve a snapshot older than the moment
/// the user explicitly asked for "latest". The baseline forces the
/// server to skip any cache entry older than the checkout time, so the
/// `checkout_latest()` signal is preserved across reads on the same
/// handle regardless of the configured consistency interval.
checkout_baseline: Option<SystemTime>,
/// This is what provides read-your-write on a single handle: after a write
/// the next read forces the server cache past the write time. It also
/// preserves the `checkout_latest()` signal when `read_consistency_interval`
/// is unset (the default), where there is no interval-based floor.
freshness_baseline: Option<SystemTime>,
}
/// Snapshot of the headers that should be attached to a single read request.
#[derive(Debug, Default, Clone, Copy)]
struct FreshnessHeaders {
min_version: Option<u64>,
min_timestamp: Option<SystemTime>,
}
impl FreshnessHeaders {
fn apply(self, mut request: RequestBuilder) -> RequestBuilder {
if let Some(v) = self.min_version {
request = request.header(MIN_VERSION_HEADER, v.to_string());
}
if let Some(ts) = self.min_timestamp {
let dt: chrono::DateTime<chrono::Utc> = ts.into();
request = request.header(MIN_TIMESTAMP_HEADER, dt.to_rfc3339());
@@ -115,6 +104,14 @@ impl FreshnessHeaders {
}
}
/// Monotonic floor for the freshness baseline. `SystemTime` is not monotonic
/// (NTP steps, hibernate/resume can move it backward), so a write must never
/// lower the baseline below a prior write's — that would let the next read send
/// a `min_timestamp` earlier than an earlier write and break read-your-write.
fn next_freshness_baseline(prev: Option<SystemTime>, now: SystemTime) -> SystemTime {
prev.map_or(now, |prev| prev.max(now))
}
fn compute_min_timestamp(
state: &FreshnessState,
interval: Option<Duration>,
@@ -125,7 +122,7 @@ fn compute_min_timestamp(
Some(d) if d.is_zero() => Some(now),
Some(d) => Some(now.checked_sub(d).unwrap_or(now)),
};
match (interval_based, state.checkout_baseline) {
match (interval_based, state.freshness_baseline) {
(None, None) => None,
(Some(t), None) | (None, Some(t)) => Some(t),
(Some(a), Some(b)) => Some(a.max(b)),
@@ -787,7 +784,6 @@ impl<S: HttpSend> RemoteTable<S> {
fn snapshot_freshness_headers(&self) -> FreshnessHeaders {
let state = *self.freshness.lock().unwrap();
FreshnessHeaders {
min_version: state.min_version,
min_timestamp: compute_min_timestamp(
&state,
self.client.read_consistency_interval,
@@ -796,22 +792,20 @@ impl<S: HttpSend> RemoteTable<S> {
}
}
/// Build a POST request and attach the read-freshness headers
/// (`x-lancedb-min-version`, `x-lancedb-min-timestamp`).
/// Build a POST request and attach the `x-lancedb-min-timestamp` freshness
/// header.
fn post_read(&self, uri: &str) -> RequestBuilder {
self.snapshot_freshness_headers()
.apply(self.client.post(uri))
}
/// Record a version returned by a write so subsequent reads can request at
/// least that version via `x-lancedb-min-version`. A returned `0` from a
/// backward-compatible old server is ignored.
fn track_write_version(&self, version: u64) {
if version == 0 {
return;
}
/// Record that this handle just performed a write, so the next read forces
/// the server cache past the write time (read-your-write on a single
/// handle).
fn bump_freshness_baseline(&self) {
let now = SystemTime::now();
let mut state = self.freshness.lock().unwrap();
state.min_version = Some(state.min_version.map_or(version, |v| v.max(version)));
state.freshness_baseline = Some(next_freshness_baseline(state.freshness_baseline, now));
}
async fn execute_query(
@@ -1115,7 +1109,7 @@ impl<S: HttpSend + 'static> RemoteTable<S> {
if output.overwrite {
self.invalidate_schema_cache();
}
self.track_write_version(add_result.version);
self.bump_freshness_baseline();
return Ok(add_result);
}
@@ -1153,7 +1147,7 @@ impl<S: HttpSend + 'static> RemoteTable<S> {
if output.overwrite {
self.invalidate_schema_cache();
}
self.track_write_version(result.version);
self.bump_freshness_baseline();
return Ok(result);
}
Err(e) => {
@@ -1271,7 +1265,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
}
async fn checkout(&self, version: u64) -> Result<()> {
// Validate the version exists. The describe is sent without freshness
// headers so a stale `min_version` from a previous write doesn't ride
// headers so a stale baseline from a previous write doesn't ride
// along on an explicit time-travel request.
let request = self
.client
@@ -1306,11 +1300,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
*write_guard = None;
drop(write_guard);
// Drop any per-handle write tracking; subsequent reads use the
// baseline timestamp captured now to guarantee freshness.
// Reset the freshness baseline to now so subsequent reads see at least
// the state as of this explicit `checkout_latest()`.
*self.freshness.lock().unwrap() = FreshnessState {
min_version: None,
checkout_baseline: Some(SystemTime::now()),
freshness_baseline: Some(SystemTime::now()),
};
// Invalidate schema cache since we're switching versions
@@ -1627,7 +1620,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
status_code: None,
})?;
self.track_write_version(update_response.version);
self.bump_freshness_baseline();
Ok(update_response)
}
@@ -1658,7 +1651,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
request_id,
status_code: None,
})?;
self.track_write_version(delete_response.version);
self.bump_freshness_baseline();
Ok(delete_response)
}
@@ -1815,7 +1808,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
status_code: None,
})?;
self.track_write_version(merge_insert_response.version);
self.bump_freshness_baseline();
Ok(merge_insert_response)
}
@@ -1842,7 +1835,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
}
async fn checkout_tag(&self, tag: &str) -> Result<()> {
// Resolve the tag without attaching freshness headers; a stale
// `min_version` from a previous write should not ride along on an
// baseline from a previous write should not ride along on an
// explicit time-travel request.
let request = self
.client
@@ -1907,7 +1900,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})?;
self.invalidate_schema_cache();
self.track_write_version(result.version);
self.bump_freshness_baseline();
Ok(result)
}
@@ -1962,7 +1955,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})?;
self.invalidate_schema_cache();
self.track_write_version(result.version);
self.bump_freshness_baseline();
Ok(result)
}
@@ -1990,7 +1983,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})?;
self.invalidate_schema_cache();
self.track_write_version(result.version);
self.bump_freshness_baseline();
Ok(result)
}
@@ -6167,6 +6160,21 @@ mod tests {
// ---- Read freshness header tests ------------------------------------
#[test]
fn test_next_freshness_baseline_is_monotonic() {
let t100 = SystemTime::UNIX_EPOCH + Duration::from_secs(100);
let t101 = SystemTime::UNIX_EPOCH + Duration::from_secs(101);
let t99 = SystemTime::UNIX_EPOCH + Duration::from_secs(99);
// No prior baseline -> take the current time.
assert_eq!(next_freshness_baseline(None, t100), t100);
// Clock moved forward -> advance to it.
assert_eq!(next_freshness_baseline(Some(t100), t101), t101);
// Clock moved backward (NTP step / resume) -> keep the higher prior
// baseline so read-your-write isn't lowered below an earlier write.
assert_eq!(next_freshness_baseline(Some(t100), t99), t100);
}
#[test]
fn test_compute_min_timestamp_combines_baseline_and_interval() {
let now = SystemTime::now();
@@ -6180,8 +6188,7 @@ mod tests {
// Baseline only -> baseline.
let state = FreshnessState {
min_version: None,
checkout_baseline: Some(baseline),
freshness_baseline: Some(baseline),
};
assert_eq!(compute_min_timestamp(&state, None, now), Some(baseline));
@@ -6204,8 +6211,7 @@ mod tests {
// Both: pick the more-recent (i.e. tighter) constraint.
// baseline = now-60, now-interval = now-10. now-10 is newer.
let state = FreshnessState {
min_version: None,
checkout_baseline: Some(baseline),
freshness_baseline: Some(baseline),
};
assert_eq!(
compute_min_timestamp(&state, Some(Duration::from_secs(10)), now),
@@ -6215,8 +6221,7 @@ mod tests {
// Both, baseline newer: pick baseline.
let recent_baseline = now - Duration::from_secs(5);
let state = FreshnessState {
min_version: None,
checkout_baseline: Some(recent_baseline),
freshness_baseline: Some(recent_baseline),
};
assert_eq!(
compute_min_timestamp(&state, Some(Duration::from_secs(60)), now),
@@ -6271,7 +6276,6 @@ mod tests {
let headers = captured.lock().unwrap().clone().unwrap();
assert!(!headers.contains_key("x-lancedb-min-timestamp"));
assert!(!headers.contains_key("x-lancedb-min-version"));
}
#[tokio::test]
@@ -6290,7 +6294,6 @@ mod tests {
sent >= before - FRESHNESS_TOLERANCE && sent <= after + FRESHNESS_TOLERANCE,
"expected timestamp roughly equal to wall clock"
);
assert!(!headers.contains_key("x-lancedb-min-version"));
}
#[tokio::test]
@@ -6334,11 +6337,10 @@ mod tests {
&& sent <= after_checkout + FRESHNESS_TOLERANCE,
"expected timestamp captured at checkout_latest() time"
);
assert!(!headers.contains_key("x-lancedb-min-version"));
}
#[tokio::test]
async fn test_freshness_min_version_tracked_after_write() {
async fn test_freshness_baseline_bumped_after_write() {
let (handler, captured) = capturing_handler(|path| match path {
"/v1/table/my_table/update/" => r#"{"rows_updated":1,"version":7}"#.to_string(),
"/v1/table/my_table/count_rows/" => "42".to_string(),
@@ -6346,19 +6348,17 @@ mod tests {
});
let table = Table::new_with_handler("my_table", handler);
let before = SystemTime::now();
let _ = table.update().column("a", "a + 1").execute().await.unwrap();
// Update headers also pass through captured; reset by reading after.
table.count_rows(None).await.unwrap();
let after = SystemTime::now();
// The write bumped the freshness baseline, so the next read carries a
// min-timestamp at or after the write time (read-your-write).
let headers = captured.lock().unwrap().clone().unwrap();
assert_eq!(
headers
.get("x-lancedb-min-version")
.unwrap()
.to_str()
.unwrap(),
"7"
);
let ts = parse_min_timestamp(&headers);
assert!(ts >= before - FRESHNESS_TOLERANCE);
assert!(ts <= after + FRESHNESS_TOLERANCE);
}
/// Like `capturing_handler`, but keeps a per-path snapshot of the headers
@@ -6391,9 +6391,10 @@ mod tests {
}
#[tokio::test]
async fn test_freshness_checkout_validation_sends_no_min_version() {
// After a write bumps min_version, calling checkout(v) must not let
// that stale header ride along on the validating /describe/ request.
async fn test_freshness_checkout_validation_sends_no_freshness_headers() {
// After a write bumps the baseline, calling checkout(v) must not let
// that stale freshness header ride along on the validating /describe/
// request.
let (handler, captured) = path_capturing_handler(|path| match path {
"/v1/table/my_table/update/" => r#"{"rows_updated":1,"version":7}"#.to_string(),
"/v1/table/my_table/describe/" => r#"{"version":5,"schema":{"fields":[]}}"#.to_string(),
@@ -6409,16 +6410,15 @@ mod tests {
.get("/v1/table/my_table/describe/")
.expect("describe should have been called by checkout(v)");
assert!(
!describe_headers.contains_key("x-lancedb-min-version"),
"checkout(v) describe must not carry stale min_version",
!describe_headers.contains_key("x-lancedb-min-timestamp"),
"checkout(v) describe must not carry a stale freshness baseline",
);
assert!(!describe_headers.contains_key("x-lancedb-min-timestamp"));
}
#[tokio::test]
async fn test_freshness_checkout_tag_resolve_sends_no_min_version() {
async fn test_freshness_checkout_tag_resolve_sends_no_freshness_headers() {
// Same invariant for checkout_tag: the tag-resolve request must not
// pick up a stale min_version from a prior write.
// pick up a stale freshness baseline from a prior write.
let (handler, captured) = path_capturing_handler(|path| match path {
"/v1/table/my_table/update/" => r#"{"rows_updated":1,"version":7}"#.to_string(),
"/v1/table/my_table/tags/version/" => r#"{"version":5}"#.to_string(),
@@ -6434,14 +6434,13 @@ mod tests {
.get("/v1/table/my_table/tags/version/")
.expect("tags/version should have been called by checkout_tag");
assert!(
!resolve_headers.contains_key("x-lancedb-min-version"),
"checkout_tag resolve must not carry stale min_version",
!resolve_headers.contains_key("x-lancedb-min-timestamp"),
"checkout_tag resolve must not carry a stale freshness baseline",
);
assert!(!resolve_headers.contains_key("x-lancedb-min-timestamp"));
}
#[tokio::test]
async fn test_freshness_checkout_clears_min_version() {
async fn test_freshness_checkout_clears_baseline() {
let (handler, captured) = capturing_handler(|path| match path {
"/v1/table/my_table/update/" => r#"{"rows_updated":1,"version":7}"#.to_string(),
// checkout(5) needs to describe version 5 first
@@ -6456,7 +6455,6 @@ mod tests {
table.count_rows(None).await.unwrap();
let headers = captured.lock().unwrap().clone().unwrap();
assert!(!headers.contains_key("x-lancedb-min-version"));
assert!(!headers.contains_key("x-lancedb-min-timestamp"));
}
}