mirror of
https://github.com/lancedb/lancedb.git
synced 2026-06-22 05:30:39 +00:00
Compare commits
4 Commits
python-v0.
...
python-v0.
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3fbfbfb702 | ||
|
|
64978c8419 | ||
|
|
241420239b | ||
|
|
9d67ea2bb0 |
@@ -29,7 +29,3 @@ runs:
|
|||||||
args: ${{ inputs.args }}
|
args: ${{ inputs.args }}
|
||||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||||
working-directory: python
|
working-directory: python
|
||||||
- uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: windows-wheels
|
|
||||||
path: python\target\wheels
|
|
||||||
|
|||||||
1
.github/workflows/java.yml
vendored
1
.github/workflows/java.yml
vendored
@@ -16,6 +16,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
paths:
|
paths:
|
||||||
- java/**
|
- java/**
|
||||||
- .github/workflows/java.yml
|
- .github/workflows/java.yml
|
||||||
|
|||||||
1
.github/workflows/license-header-check.yml
vendored
1
.github/workflows/license-header-check.yml
vendored
@@ -3,6 +3,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- rust/**
|
- rust/**
|
||||||
|
|||||||
1
.github/workflows/nodejs.yml
vendored
1
.github/workflows/nodejs.yml
vendored
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
|
|||||||
100
.github/workflows/pypi-publish.yml
vendored
100
.github/workflows/pypi-publish.yml
vendored
@@ -8,6 +8,9 @@ on:
|
|||||||
# This should trigger a dry run (we skip the final publish step)
|
# This should trigger a dry run (we skip the final publish step)
|
||||||
paths:
|
paths:
|
||||||
- .github/workflows/pypi-publish.yml
|
- .github/workflows/pypi-publish.yml
|
||||||
|
- .github/workflows/build_linux_wheel/action.yml
|
||||||
|
- .github/workflows/build_mac_wheel/action.yml
|
||||||
|
- .github/workflows/build_windows_wheel/action.yml
|
||||||
- Cargo.toml # Change in dependency frequently breaks builds
|
- Cargo.toml # Change in dependency frequently breaks builds
|
||||||
- Cargo.lock
|
- Cargo.lock
|
||||||
|
|
||||||
@@ -21,9 +24,6 @@ jobs:
|
|||||||
linux:
|
linux:
|
||||||
name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
|
||||||
timeout-minutes: 60
|
timeout-minutes: 60
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
config:
|
config:
|
||||||
@@ -46,7 +46,7 @@ jobs:
|
|||||||
runner: ubuntu-2404-8x-arm64
|
runner: ubuntu-2404-8x-arm64
|
||||||
runs-on: ${{ matrix.config.runner }}
|
runs-on: ${{ matrix.config.runner }}
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v6
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
@@ -60,15 +60,14 @@ jobs:
|
|||||||
args: "--release --strip ${{ matrix.config.extra_args }}"
|
args: "--release --strip ${{ matrix.config.extra_args }}"
|
||||||
arm-build: ${{ matrix.config.platform == 'aarch64' }}
|
arm-build: ${{ matrix.config.platform == 'aarch64' }}
|
||||||
manylinux: ${{ matrix.config.manylinux }}
|
manylinux: ${{ matrix.config.manylinux }}
|
||||||
- uses: ./.github/workflows/upload_wheel
|
- uses: actions/upload-artifact@v7
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
with:
|
with:
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
name: wheels-linux-${{ matrix.config.platform }}-${{ matrix.config.manylinux }}
|
||||||
|
path: target/wheels/lancedb-*.whl
|
||||||
|
if-no-files-found: error
|
||||||
mac:
|
mac:
|
||||||
timeout-minutes: 90
|
timeout-minutes: 90
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
runs-on: ${{ matrix.config.runner }}
|
runs-on: ${{ matrix.config.runner }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -78,7 +77,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
MACOSX_DEPLOYMENT_TARGET: 10.15
|
MACOSX_DEPLOYMENT_TARGET: 10.15
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v6
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
@@ -90,18 +89,21 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
python-minor-version: 10
|
python-minor-version: 10
|
||||||
args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
|
args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
|
||||||
- uses: ./.github/workflows/upload_wheel
|
- uses: actions/upload-artifact@v7
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
with:
|
with:
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
name: wheels-mac-${{ matrix.config.target }}
|
||||||
|
path: target/wheels/lancedb-*.whl
|
||||||
|
if-no-files-found: error
|
||||||
windows:
|
windows:
|
||||||
timeout-minutes: 60
|
timeout-minutes: 90
|
||||||
permissions:
|
|
||||||
id-token: write
|
|
||||||
contents: read
|
|
||||||
runs-on: windows-latest
|
runs-on: windows-latest
|
||||||
|
env:
|
||||||
|
# link.exe is single-threaded and the long pole on Windows builds. Use
|
||||||
|
# rustc's bundled lld-link instead.
|
||||||
|
CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: rust-lld
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v6
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
@@ -113,18 +115,70 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
python-minor-version: 10
|
python-minor-version: 10
|
||||||
args: "--release --strip"
|
args: "--release --strip"
|
||||||
vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
|
- uses: actions/upload-artifact@v7
|
||||||
- uses: ./.github/workflows/upload_wheel
|
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
with:
|
with:
|
||||||
fury_token: ${{ secrets.FURY_TOKEN }}
|
name: wheels-windows
|
||||||
|
path: target/wheels/lancedb-*.whl
|
||||||
|
if-no-files-found: error
|
||||||
|
publish:
|
||||||
|
name: Publish wheels
|
||||||
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
|
needs: [linux, mac, windows]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
id-token: write
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v6
|
||||||
|
- name: Download wheel artifacts
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
pattern: wheels-*
|
||||||
|
path: target/wheels
|
||||||
|
merge-multiple: true
|
||||||
|
- name: List wheels
|
||||||
|
run: ls -la target/wheels
|
||||||
|
- name: Choose repo
|
||||||
|
id: choose_repo
|
||||||
|
run: |
|
||||||
|
if [[ ${{ github.ref }} == *beta* ]]; then
|
||||||
|
echo "repo=fury" >> $GITHUB_OUTPUT
|
||||||
|
else
|
||||||
|
echo "repo=pypi" >> $GITHUB_OUTPUT
|
||||||
|
fi
|
||||||
|
- name: Publish to Fury
|
||||||
|
if: steps.choose_repo.outputs.repo == 'fury'
|
||||||
|
env:
|
||||||
|
FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
|
||||||
|
run: |
|
||||||
|
shopt -s nullglob
|
||||||
|
WHEELS=(target/wheels/lancedb-*.whl)
|
||||||
|
if [[ ${#WHEELS[@]} -eq 0 ]]; then
|
||||||
|
echo "No wheels found in target/wheels/" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for WHEEL in "${WHEELS[@]}"; do
|
||||||
|
echo "Uploading $WHEEL to Fury"
|
||||||
|
curl -f -F package=@"$WHEEL" "https://$FURY_TOKEN@push.fury.io/lancedb/"
|
||||||
|
done
|
||||||
|
# NOTE: pypa/gh-action-pypi-publish must be invoked directly from a
|
||||||
|
# workflow file, not from inside a composite action. When called from a
|
||||||
|
# composite, `github.action_repository` is empty (actions/runner#2473)
|
||||||
|
# and the action falls back to `github.repository`, producing a bogus
|
||||||
|
# `docker://ghcr.io/<repo>:<ref>` image reference that GHA tries to pull.
|
||||||
|
- name: Publish to PyPI
|
||||||
|
if: steps.choose_repo.outputs.repo == 'pypi'
|
||||||
|
uses: pypa/gh-action-pypi-publish@release/v1
|
||||||
|
with:
|
||||||
|
packages-dir: target/wheels/
|
||||||
gh-release:
|
gh-release:
|
||||||
if: startsWith(github.ref, 'refs/tags/python-v')
|
if: startsWith(github.ref, 'refs/tags/python-v')
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
permissions:
|
permissions:
|
||||||
contents: write
|
contents: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v6
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
lfs: true
|
lfs: true
|
||||||
@@ -187,13 +241,13 @@ jobs:
|
|||||||
report-failure:
|
report-failure:
|
||||||
name: Report Workflow Failure
|
name: Report Workflow Failure
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [linux, mac, windows]
|
needs: [linux, mac, windows, publish]
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
issues: write
|
issues: write
|
||||||
if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
|
if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v6
|
||||||
- uses: ./.github/actions/create-failure-issue
|
- uses: ./.github/actions/create-failure-issue
|
||||||
with:
|
with:
|
||||||
job-results: ${{ toJSON(needs) }}
|
job-results: ${{ toJSON(needs) }}
|
||||||
|
|||||||
1
.github/workflows/python.yml
vendored
1
.github/workflows/python.yml
vendored
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
|
|||||||
1
.github/workflows/rust.yml
vendored
1
.github/workflows/rust.yml
vendored
@@ -4,6 +4,7 @@ on:
|
|||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
|
- release/**
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- Cargo.toml
|
- Cargo.toml
|
||||||
|
|||||||
34
.github/workflows/upload_wheel/action.yml
vendored
34
.github/workflows/upload_wheel/action.yml
vendored
@@ -1,34 +0,0 @@
|
|||||||
name: upload-wheel
|
|
||||||
|
|
||||||
description: "Upload wheels to Pypi"
|
|
||||||
inputs:
|
|
||||||
fury_token:
|
|
||||||
required: true
|
|
||||||
description: "release token for the fury repo"
|
|
||||||
|
|
||||||
runs:
|
|
||||||
using: "composite"
|
|
||||||
steps:
|
|
||||||
- name: Choose repo
|
|
||||||
shell: bash
|
|
||||||
id: choose_repo
|
|
||||||
run: |
|
|
||||||
if [[ ${{ github.ref }} == *beta* ]]; then
|
|
||||||
echo "repo=fury" >> $GITHUB_OUTPUT
|
|
||||||
else
|
|
||||||
echo "repo=pypi" >> $GITHUB_OUTPUT
|
|
||||||
fi
|
|
||||||
- name: Publish to Fury
|
|
||||||
if: steps.choose_repo.outputs.repo == 'fury'
|
|
||||||
shell: bash
|
|
||||||
env:
|
|
||||||
FURY_TOKEN: ${{ inputs.fury_token }}
|
|
||||||
run: |
|
|
||||||
WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1)
|
|
||||||
echo "Uploading $WHEEL to Fury"
|
|
||||||
curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/
|
|
||||||
- name: Publish to PyPI
|
|
||||||
if: steps.choose_repo.outputs.repo == 'pypi'
|
|
||||||
uses: pypa/gh-action-pypi-publish@release/v1
|
|
||||||
with:
|
|
||||||
packages-dir: target/wheels/
|
|
||||||
1425
Cargo.lock
generated
1425
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
30
Cargo.toml
30
Cargo.toml
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
|||||||
rust-version = "1.91.0"
|
rust-version = "1.91.0"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
lance = { "version" = "=7.0.0-beta.7", default-features = false, "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance = { "version" = "=6.0.0", default-features = false }
|
||||||
lance-core = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-core = "=6.0.0"
|
||||||
lance-datagen = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datagen = "=6.0.0"
|
||||||
lance-file = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-file = "=6.0.0"
|
||||||
lance-io = { "version" = "=7.0.0-beta.7", default-features = false, "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-io = { "version" = "=6.0.0", default-features = false }
|
||||||
lance-index = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-index = "=6.0.0"
|
||||||
lance-linalg = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-linalg = "=6.0.0"
|
||||||
lance-namespace = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace = "=6.0.0"
|
||||||
lance-namespace-impls = { "version" = "=7.0.0-beta.7", default-features = false, "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace-impls = { "version" = "=6.0.0", default-features = false }
|
||||||
lance-table = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-table = "=6.0.0"
|
||||||
lance-testing = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-testing = "=6.0.0"
|
||||||
lance-datafusion = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datafusion = "=6.0.0"
|
||||||
lance-encoding = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-encoding = "=6.0.0"
|
||||||
lance-arrow = { "version" = "=7.0.0-beta.7", "tag" = "v7.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
lance-arrow = "=6.0.0"
|
||||||
ahash = "0.8"
|
ahash = "0.8"
|
||||||
# Note that this one does not include pyarrow
|
# Note that this one does not include pyarrow
|
||||||
arrow = { version = "58.0.0", optional = false }
|
arrow = { version = "58.0.0", optional = false }
|
||||||
@@ -54,7 +54,7 @@ half = { "version" = "2.7.1", default-features = false, features = [
|
|||||||
futures = "0"
|
futures = "0"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
moka = { version = "0.12", features = ["future"] }
|
moka = { version = "0.12", features = ["future"] }
|
||||||
object_store = "0.13.2"
|
object_store = "0.12.0"
|
||||||
pin-project = "1.0.7"
|
pin-project = "1.0.7"
|
||||||
rand = "0.9"
|
rand = "0.9"
|
||||||
snafu = "0.8"
|
snafu = "0.8"
|
||||||
|
|||||||
@@ -28,7 +28,7 @@
|
|||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<arrow.version>15.0.0</arrow.version>
|
<arrow.version>15.0.0</arrow.version>
|
||||||
<lance-core.version>7.0.0-beta.7</lance-core.version>
|
<lance-core.version>6.0.0</lance-core.version>
|
||||||
<spotless.skip>false</spotless.skip>
|
<spotless.skip>false</spotless.skip>
|
||||||
<spotless.version>2.30.0</spotless.version>
|
<spotless.version>2.30.0</spotless.version>
|
||||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.31.0-beta.11"
|
current_version = "0.32.0"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.31.0-beta.11"
|
version = "0.32.0"
|
||||||
publish = false
|
publish = false
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"
|
|||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
pylance = [
|
pylance = [
|
||||||
"pylance>=5.0.0b5",
|
"pylance>=6.0.0",
|
||||||
]
|
]
|
||||||
tests = [
|
tests = [
|
||||||
"aiohttp>=3.9.0",
|
"aiohttp>=3.9.0",
|
||||||
@@ -58,7 +58,7 @@ tests = [
|
|||||||
"pytz>=2023.3",
|
"pytz>=2023.3",
|
||||||
"polars>=0.19, <=1.3.0",
|
"polars>=0.19, <=1.3.0",
|
||||||
"pyarrow-stubs>=16.0",
|
"pyarrow-stubs>=16.0",
|
||||||
"pylance>=5.0.0b5",
|
"pylance>=6.0.0",
|
||||||
"requests>=2.31.0",
|
"requests>=2.31.0",
|
||||||
"datafusion>=52,<53",
|
"datafusion>=52,<53",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -108,12 +108,7 @@ test-log = "0.2"
|
|||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
aws = [
|
aws = ["lance/aws", "lance-io/aws", "lance-namespace-impls/dir-aws"]
|
||||||
"lance/aws",
|
|
||||||
"lance-io/aws",
|
|
||||||
"lance-namespace-impls/dir-aws",
|
|
||||||
"object_store/aws",
|
|
||||||
]
|
|
||||||
oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
|
oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
|
||||||
gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
|
gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
|
||||||
azure = [
|
azure = [
|
||||||
|
|||||||
@@ -722,7 +722,7 @@ impl ListingDatabase {
|
|||||||
let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params)).await?;
|
let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params)).await?;
|
||||||
for name in names {
|
for name in names {
|
||||||
let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
|
let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
|
||||||
let full_path = self.base_path.clone().join(dir_name.clone());
|
let full_path = self.base_path.child(dir_name.clone());
|
||||||
|
|
||||||
commit_handler.delete(&full_path).await?;
|
commit_handler.delete(&full_path).await?;
|
||||||
|
|
||||||
|
|||||||
@@ -5,12 +5,11 @@
|
|||||||
|
|
||||||
use std::{fmt::Formatter, sync::Arc};
|
use std::{fmt::Formatter, sync::Arc};
|
||||||
|
|
||||||
use futures::{StreamExt, TryFutureExt, stream::BoxStream};
|
use futures::{TryFutureExt, stream::BoxStream};
|
||||||
use lance::io::WrappingObjectStore;
|
use lance::io::WrappingObjectStore;
|
||||||
use object_store::{
|
use object_store::{
|
||||||
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
|
Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||||
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
|
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, path::Path,
|
||||||
UploadPart, path::Path,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
@@ -94,6 +93,20 @@ impl ObjectStore for MirroringObjectStore {
|
|||||||
self.primary.get_opts(location, options).await
|
self.primary.get_opts(location, options).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn head(&self, location: &Path) -> Result<ObjectMeta> {
|
||||||
|
self.primary.head(location).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn delete(&self, location: &Path) -> Result<()> {
|
||||||
|
if !location.primary_only() {
|
||||||
|
match self.secondary.delete(location).await {
|
||||||
|
Err(Error::NotFound { .. }) | Ok(_) => {}
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.primary.delete(location).await
|
||||||
|
}
|
||||||
|
|
||||||
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
|
||||||
self.primary.list(prefix)
|
self.primary.list(prefix)
|
||||||
}
|
}
|
||||||
@@ -102,41 +115,22 @@ impl ObjectStore for MirroringObjectStore {
|
|||||||
self.primary.list_with_delimiter(prefix).await
|
self.primary.list_with_delimiter(prefix).await
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete_stream(
|
async fn copy(&self, from: &Path, to: &Path) -> Result<()> {
|
||||||
&self,
|
|
||||||
locations: BoxStream<'static, Result<Path>>,
|
|
||||||
) -> BoxStream<'static, Result<Path>> {
|
|
||||||
let primary = self.primary.clone();
|
|
||||||
let secondary = self.secondary.clone();
|
|
||||||
locations
|
|
||||||
.map(move |location| {
|
|
||||||
let primary = primary.clone();
|
|
||||||
let secondary = secondary.clone();
|
|
||||||
async move {
|
|
||||||
let location = location?;
|
|
||||||
if !location.primary_only() {
|
|
||||||
match secondary.delete(&location).await {
|
|
||||||
Err(Error::NotFound { .. }) | Ok(_) => {}
|
|
||||||
Err(e) => return Err(e),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
primary.delete(&location).await?;
|
|
||||||
Ok(location)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.buffered(10)
|
|
||||||
.boxed()
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
|
|
||||||
if to.primary_only() {
|
if to.primary_only() {
|
||||||
self.primary.copy_opts(from, to, options).await
|
self.primary.copy(from, to).await
|
||||||
} else {
|
} else {
|
||||||
self.secondary.copy_opts(from, to, options.clone()).await?;
|
self.secondary.copy(from, to).await?;
|
||||||
self.primary.copy_opts(from, to, options).await?;
|
self.primary.copy(from, to).await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> {
|
||||||
|
if !to.primary_only() {
|
||||||
|
self.secondary.copy(from, to).await?;
|
||||||
|
}
|
||||||
|
self.primary.copy_if_not_exists(from, to).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
|
|||||||
@@ -10,9 +10,9 @@ use bytes::Bytes;
|
|||||||
use futures::stream::BoxStream;
|
use futures::stream::BoxStream;
|
||||||
use lance::io::WrappingObjectStore;
|
use lance::io::WrappingObjectStore;
|
||||||
use object_store::{
|
use object_store::{
|
||||||
CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
|
||||||
PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
|
PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as OSResult, UploadPart,
|
||||||
UploadPart, path::Path,
|
path::Path,
|
||||||
};
|
};
|
||||||
|
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
@@ -81,6 +81,11 @@ impl IoTrackingStore {
|
|||||||
#[async_trait::async_trait]
|
#[async_trait::async_trait]
|
||||||
#[deny(clippy::missing_trait_methods)]
|
#[deny(clippy::missing_trait_methods)]
|
||||||
impl ObjectStore for IoTrackingStore {
|
impl ObjectStore for IoTrackingStore {
|
||||||
|
async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
|
||||||
|
self.record_write(bytes.content_length() as u64);
|
||||||
|
self.target.put(location, bytes).await
|
||||||
|
}
|
||||||
|
|
||||||
async fn put_opts(
|
async fn put_opts(
|
||||||
&self,
|
&self,
|
||||||
location: &Path,
|
location: &Path,
|
||||||
@@ -91,6 +96,14 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
self.target.put_opts(location, bytes, opts).await
|
self.target.put_opts(location, bytes, opts).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
|
||||||
|
let target = self.target.put_multipart(location).await?;
|
||||||
|
Ok(Box::new(IoTrackingMultipartUpload {
|
||||||
|
target,
|
||||||
|
stats: self.stats.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
async fn put_multipart_opts(
|
async fn put_multipart_opts(
|
||||||
&self,
|
&self,
|
||||||
location: &Path,
|
location: &Path,
|
||||||
@@ -103,6 +116,15 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get(&self, location: &Path) -> OSResult<GetResult> {
|
||||||
|
let result = self.target.get(location).await;
|
||||||
|
if let Ok(result) = &result {
|
||||||
|
let num_bytes = result.range.end - result.range.start;
|
||||||
|
self.record_read(num_bytes);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
|
async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
|
||||||
let result = self.target.get_opts(location, options).await;
|
let result = self.target.get_opts(location, options).await;
|
||||||
if let Ok(result) = &result {
|
if let Ok(result) = &result {
|
||||||
@@ -112,6 +134,14 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn get_range(&self, location: &Path, range: std::ops::Range<u64>) -> OSResult<Bytes> {
|
||||||
|
let result = self.target.get_range(location, range).await;
|
||||||
|
if let Ok(result) = &result {
|
||||||
|
self.record_read(result.len() as u64);
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
async fn get_ranges(
|
async fn get_ranges(
|
||||||
&self,
|
&self,
|
||||||
location: &Path,
|
location: &Path,
|
||||||
@@ -124,11 +154,20 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete_stream(
|
async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
|
||||||
&self,
|
self.record_read(0);
|
||||||
locations: BoxStream<'static, OSResult<Path>>,
|
self.target.head(location).await
|
||||||
) -> BoxStream<'static, OSResult<Path>> {
|
}
|
||||||
|
|
||||||
|
async fn delete(&self, location: &Path) -> OSResult<()> {
|
||||||
self.record_write(0);
|
self.record_write(0);
|
||||||
|
self.target.delete(location).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn delete_stream<'a>(
|
||||||
|
&'a self,
|
||||||
|
locations: BoxStream<'a, OSResult<Path>>,
|
||||||
|
) -> BoxStream<'a, OSResult<Path>> {
|
||||||
self.target.delete_stream(locations)
|
self.target.delete_stream(locations)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -151,14 +190,24 @@ impl ObjectStore for IoTrackingStore {
|
|||||||
self.target.list_with_delimiter(prefix).await
|
self.target.list_with_delimiter(prefix).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> OSResult<()> {
|
async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
self.record_write(0);
|
self.record_write(0);
|
||||||
self.target.copy_opts(from, to, options).await
|
self.target.copy(from, to).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> OSResult<()> {
|
async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
self.record_write(0);
|
self.record_write(0);
|
||||||
self.target.rename_opts(from, to, options).await
|
self.target.rename(from, to).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
|
self.record_write(0);
|
||||||
|
self.target.rename_if_not_exists(from, to).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
|
||||||
|
self.record_write(0);
|
||||||
|
self.target.copy_if_not_exists(from, to).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user