mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-27 08:28:28 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5e58dde32d |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.32.0-beta.0"
|
||||
current_version = "0.31.0-beta.6"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
# CODEOWNERS
|
||||
#
|
||||
# These owners will be the default owners for everything in the repo.
|
||||
# They will be requested for review when someone opens a pull request.
|
||||
#
|
||||
# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
|
||||
|
||||
# Default owners for everything
|
||||
* @jackye1995 @wjones127
|
||||
|
||||
# Release and publish workflows — changes here can affect supply chain security
|
||||
/.github/workflows/ @jackye1995 @wjones127 @Xuanwo
|
||||
|
||||
# Remote client and auth — sensitive networking and auth code
|
||||
/rust/lancedb/src/remote/ @jackye1995 @wjones127
|
||||
|
||||
# Python FFI boundary
|
||||
/python/src/ @jackye1995 @wjones127 @AyushExel
|
||||
|
||||
# NodeJS FFI boundary
|
||||
/nodejs/src/ @jackye1995 @wjones127
|
||||
@@ -34,16 +34,15 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||
target: x86_64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
|
||||
&& unzip /tmp/protoc.zip -d /usr/local \
|
||||
&& rm /tmp/protoc.zip
|
||||
- name: Build Arm Manylinux Wheel
|
||||
if: ${{ inputs.arm-build == 'true' }}
|
||||
uses: PyO3/maturin-action@v1
|
||||
@@ -51,14 +50,13 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||
target: aarch64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
yum install -y clang
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
yum install -y clang \
|
||||
&& curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
|
||||
&& unzip /tmp/protoc.zip -d /usr/local \
|
||||
&& rm /tmp/protoc.zip
|
||||
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
features: fp16kernels
|
||||
pre_build: brew install protobuf
|
||||
- target: x86_64-pc-windows-msvc
|
||||
host: windows-2025-8x-x64
|
||||
host: windows-latest
|
||||
features: ","
|
||||
pre_build: |-
|
||||
choco install --no-progress protoc ninja nasm
|
||||
@@ -111,21 +111,12 @@ jobs:
|
||||
# There is an issue where choco doesn't add nasm to the path
|
||||
export PATH="$PATH:/c/Program Files/NASM"
|
||||
nasm -v
|
||||
# Fat LTO of the cdylib is single-threaded and the peak-memory
|
||||
# step of the build, and had started hitting rustc-LLVM OOM on the
|
||||
# Windows runners. ThinLTO parallelizes it across the runner's
|
||||
# cores and keeps peak memory well under the limit.
|
||||
export CARGO_PROFILE_RELEASE_LTO=thin
|
||||
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
|
||||
- target: aarch64-pc-windows-msvc
|
||||
host: windows-2025-8x-x64
|
||||
host: windows-latest
|
||||
features: ","
|
||||
pre_build: |-
|
||||
choco install --no-progress protoc
|
||||
rustup target add aarch64-pc-windows-msvc
|
||||
# See ThinLTO note on the x86_64-pc-windows-msvc target above.
|
||||
export CARGO_PROFILE_RELEASE_LTO=thin
|
||||
export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
host: ubuntu-latest
|
||||
features: fp16kernels
|
||||
|
||||
Generated
+56
-292
@@ -2039,9 +2039,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.20"
|
||||
version = "0.9.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
@@ -3151,12 +3151,6 @@ dependencies = [
|
||||
"encoding_rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "endian-type"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
|
||||
|
||||
[[package]]
|
||||
name = "enum-as-inner"
|
||||
version = "0.6.1"
|
||||
@@ -3429,8 +3423,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.4",
|
||||
@@ -3785,30 +3779,6 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "goosefs-sdk"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"dashmap",
|
||||
"hostname",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"rand 0.9.4",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "group"
|
||||
version = "0.12.1"
|
||||
@@ -4020,17 +3990,6 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hostname"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.4",
|
||||
"libc",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.12"
|
||||
@@ -4191,19 +4150,6 @@ dependencies = [
|
||||
"webpki-roots 1.0.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-timeout"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
|
||||
dependencies = [
|
||||
"hyper 1.9.0",
|
||||
"hyper-util",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.20"
|
||||
@@ -4727,7 +4673,7 @@ dependencies = [
|
||||
"jiff",
|
||||
"nom 8.0.0",
|
||||
"num-traits",
|
||||
"ordered-float 5.3.0",
|
||||
"ordered-float",
|
||||
"rand 0.9.4",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -4780,8 +4726,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4855,8 +4801,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4865,7 +4811,6 @@ dependencies = [
|
||||
"arrow-ord",
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"bytemuck",
|
||||
"bytes",
|
||||
"futures",
|
||||
"getrandom 0.2.17",
|
||||
@@ -4878,7 +4823,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4892,7 +4837,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4901,8 +4846,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4912,8 +4857,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4951,8 +4896,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4982,8 +4927,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5000,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5010,8 +4955,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5046,8 +4991,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5077,8 +5022,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5143,8 +5088,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -5168,7 +5113,6 @@ dependencies = [
|
||||
"lance-core",
|
||||
"lance-namespace",
|
||||
"log",
|
||||
"metrics",
|
||||
"moka",
|
||||
"object_store",
|
||||
"object_store_opendal",
|
||||
@@ -5186,8 +5130,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5203,8 +5147,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5216,8 +5160,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5240,7 +5184,7 @@ dependencies = [
|
||||
"lance-table",
|
||||
"log",
|
||||
"object_store",
|
||||
"quick-xml 0.40.1",
|
||||
"quick-xml 0.38.4",
|
||||
"rand 0.9.4",
|
||||
"reqwest 0.12.28",
|
||||
"roaring",
|
||||
@@ -5271,8 +5215,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5287,8 +5231,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5327,8 +5271,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5341,8 +5285,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
version = "9.0.0-beta.16"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
@@ -5355,7 +5299,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.32.0-beta.0"
|
||||
version = "0.31.0-beta.6"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5411,8 +5355,6 @@ dependencies = [
|
||||
"lance-testing",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"moka",
|
||||
"num-traits",
|
||||
"object_store",
|
||||
@@ -5443,7 +5385,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.32.0-beta.0"
|
||||
version = "0.31.0-beta.6"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5468,7 +5410,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.35.0-beta.0"
|
||||
version = "0.34.0-beta.6"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5870,36 +5812,6 @@ dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "metrics"
|
||||
version = "0.24.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
|
||||
dependencies = [
|
||||
"portable-atomic",
|
||||
"rapidhash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "metrics-util"
|
||||
version = "0.19.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
"hashbrown 0.15.5",
|
||||
"indexmap 2.14.0",
|
||||
"metrics",
|
||||
"ordered-float 4.6.0",
|
||||
"quanta",
|
||||
"radix_trie",
|
||||
"rand 0.9.4",
|
||||
"rand_xoshiro",
|
||||
"sketches-ddsketch",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mime"
|
||||
version = "0.3.17"
|
||||
@@ -6115,15 +6027,6 @@ dependencies = [
|
||||
"rawpointer",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nibble_vec"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
|
||||
dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.26.4"
|
||||
@@ -6439,9 +6342,7 @@ dependencies = [
|
||||
"opendal-layer-timeout",
|
||||
"opendal-service-azblob",
|
||||
"opendal-service-azdls",
|
||||
"opendal-service-cos",
|
||||
"opendal-service-gcs",
|
||||
"opendal-service-goosefs",
|
||||
"opendal-service-hf",
|
||||
"opendal-service-oss",
|
||||
"opendal-service-s3",
|
||||
@@ -6569,23 +6470,6 @@ dependencies = [
|
||||
"opendal-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opendal-service-cos"
|
||||
version = "0.57.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http 1.4.2",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"quick-xml 0.39.4",
|
||||
"reqsign-core",
|
||||
"reqsign-file-read-tokio",
|
||||
"reqsign-tencent-cos",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opendal-service-gcs"
|
||||
version = "0.57.0"
|
||||
@@ -6607,20 +6491,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opendal-service-goosefs"
|
||||
version = "0.57.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"goosefs-sdk",
|
||||
"log",
|
||||
"opendal-core",
|
||||
"serde",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opendal-service-hf"
|
||||
version = "0.57.0"
|
||||
@@ -6688,15 +6558,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "5.3.0"
|
||||
@@ -7736,21 +7597,6 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quanta"
|
||||
version = "0.12.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"raw-cpuid",
|
||||
"wasi 0.11.1+wasi-snapshot-preview1",
|
||||
"web-sys",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.26.0"
|
||||
@@ -7760,6 +7606,15 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.39.4"
|
||||
@@ -7770,15 +7625,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.40.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
@@ -7862,16 +7708,6 @@ version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
|
||||
|
||||
[[package]]
|
||||
name = "radix_trie"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
|
||||
dependencies = [
|
||||
"endian-type",
|
||||
"nibble_vec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rancor"
|
||||
version = "0.1.1"
|
||||
@@ -8006,15 +7842,6 @@ version = "1.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
|
||||
|
||||
[[package]]
|
||||
name = "rapidhash"
|
||||
version = "4.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e"
|
||||
dependencies = [
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "raw-cpuid"
|
||||
version = "11.6.0"
|
||||
@@ -8299,21 +8126,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqsign-tencent-cos"
|
||||
version = "3.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"http 1.4.2",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"reqsign-core",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.12.28"
|
||||
@@ -9192,12 +9004,6 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
|
||||
|
||||
[[package]]
|
||||
name = "sketches-ddsketch"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
@@ -10005,45 +9811,6 @@ dependencies = [
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"h2 0.4.14",
|
||||
"http 1.4.2",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.9.0",
|
||||
"hyper-timeout",
|
||||
"hyper-util",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
"socket2 0.6.3",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost"
|
||||
version = "0.14.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost",
|
||||
"tonic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower"
|
||||
version = "0.5.3"
|
||||
@@ -10052,12 +9819,9 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"indexmap 2.14.0",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
|
||||
+14
-16
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=9.0.0-beta.16", default-features = false, "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=9.0.0-beta.16", default-features = false, "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=9.0.0-beta.16", default-features = false, "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
@@ -54,8 +54,6 @@ half = { "version" = "2.7.1", default-features = false, features = [
|
||||
] }
|
||||
futures = "0"
|
||||
log = "0.4"
|
||||
metrics = "0.24"
|
||||
metrics-util = "0.19"
|
||||
moka = { version = "0.12", features = ["future"] }
|
||||
object_store = "0.13.2"
|
||||
pin-project = "1.0.7"
|
||||
|
||||
@@ -51,6 +51,18 @@ ignore = [
|
||||
# https://rustsec.org/advisories/RUSTSEC-2024-0436
|
||||
{ id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
|
||||
|
||||
# encoding: unmaintained. Reached through lindera-dictionary, which is
|
||||
# required by the native Lindera tokenizer path. Lindera has not migrated
|
||||
# off this crate yet.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2021-0153
|
||||
{ id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
|
||||
|
||||
# fast-float: unsound and unmaintained. Reached only through polars-arrow
|
||||
# from the optional Polars integration; replacement requires a Polars
|
||||
# dependency upgrade.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2024-0379
|
||||
{ id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
|
||||
|
||||
# tantivy: segfault on malformed input due to missing bounds check.
|
||||
# Pulled in via lance for full-text search. We only feed tantivy
|
||||
# documents we construct ourselves, not attacker-controlled bytes.
|
||||
@@ -68,6 +80,18 @@ ignore = [
|
||||
# https://rustsec.org/advisories/RUSTSEC-2025-0119
|
||||
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
|
||||
|
||||
# bincode: unmaintained. Reached through lindera and lindera-dictionary,
|
||||
# which are required by the native Lindera tokenizer path. Lindera has not
|
||||
# migrated to another serialization format yet.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2025-0141
|
||||
{ id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
|
||||
|
||||
# lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
|
||||
# LanceDB's dev-dependency graph; LanceDB does not use that iterator
|
||||
# directly. Clearing this requires the AWS SDK chain to update lru.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0002
|
||||
{ id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
|
||||
|
||||
# rustls-webpki 0.101.7 (old major line): name-constraint checks for
|
||||
# URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
|
||||
# from aws-smithy-http-client. The 0.103 line we actively use is patched.
|
||||
@@ -84,6 +108,12 @@ ignore = [
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0104
|
||||
{ id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
|
||||
|
||||
# rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
|
||||
# logger. Reached through several transitive chains. LanceDB does not use
|
||||
# rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0097
|
||||
{ id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
|
||||
|
||||
# pyo3 advisories in the Python bindings; tracked pending a patched pyo3 release.
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0176
|
||||
# https://rustsec.org/advisories/RUSTSEC-2026-0177
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.32.0-beta.0</version>
|
||||
<version>0.31.0-beta.6</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -398,26 +398,6 @@ Drop an index from the table.
|
||||
|
||||
***
|
||||
|
||||
### getLsmWriteSpec()
|
||||
|
||||
```ts
|
||||
abstract getLsmWriteSpec(): Promise<undefined | LsmWriteSpec>
|
||||
```
|
||||
|
||||
Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table.
|
||||
|
||||
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
|
||||
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
|
||||
The returned spec — including its `maintainedIndexes` and
|
||||
`writerConfigDefaults` — mirrors what was passed to
|
||||
[Table#setLsmWriteSpec](Table.md#setlsmwritespec).
|
||||
|
||||
#### Returns
|
||||
|
||||
`Promise`<`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)>
|
||||
|
||||
***
|
||||
|
||||
### indexStats()
|
||||
|
||||
```ts
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / instrumentLanceDbMetrics
|
||||
|
||||
# Function: instrumentLanceDbMetrics()
|
||||
|
||||
```ts
|
||||
function instrumentLanceDbMetrics(meterProvider?): boolean
|
||||
```
|
||||
|
||||
Register LanceDB metrics as OpenTelemetry observable instruments.
|
||||
|
||||
Installs a process-global metrics recorder and creates one observable
|
||||
instrument per LanceDB metric (currently object store request counts, bytes,
|
||||
latency, errors, and throttles) on the given (or global) `MeterProvider`. The
|
||||
configured `MetricReader` then collects them on its own schedule.
|
||||
|
||||
Counters and gauges map directly to observable counters/gauges. Because
|
||||
OpenTelemetry has no asynchronous histogram instrument, each histogram is
|
||||
exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
|
||||
with an `le` attribute) plus `<name>_count` and `<name>_sum`.
|
||||
|
||||
Requires `@opentelemetry/api` (a dependency) and, to actually export, an
|
||||
OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
|
||||
|
||||
## Parameters
|
||||
|
||||
* **meterProvider?**: `MeterProvider`
|
||||
The provider to register instruments on. Defaults to the
|
||||
global provider from `@opentelemetry/api`.
|
||||
|
||||
## Returns
|
||||
|
||||
`boolean`
|
||||
|
||||
`true` if the recorder is installed and instruments are registered.
|
||||
`false` if a different `metrics` recorder is already installed in this
|
||||
process (only one global recorder is permitted), in which case a warning is
|
||||
emitted and no instruments are created. Calling this more than once is safe;
|
||||
instruments are created only on the first successful call.
|
||||
@@ -131,7 +131,6 @@
|
||||
- [RecordBatchIterator](functions/RecordBatchIterator.md)
|
||||
- [connect](functions/connect.md)
|
||||
- [connectNamespace](functions/connectNamespace.md)
|
||||
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
|
||||
- [makeArrowTable](functions/makeArrowTable.md)
|
||||
- [packBits](functions/packBits.md)
|
||||
- [permutationBuilder](functions/permutationBuilder.md)
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.32.0-beta.0</version>
|
||||
<version>0.31.0-beta.6</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.32.0-beta.0</version>
|
||||
<version>0.31.0-beta.6</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>9.0.0-beta.19</lance-core.version>
|
||||
<lance-core.version>9.0.0-beta.16</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.32.0-beta.0"
|
||||
version = "0.31.0-beta.6"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
@@ -44,6 +44,6 @@ aws-lc-rs = "=1.16.3"
|
||||
napi-build = "2.3.1"
|
||||
|
||||
[features]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs", "lancedb/metrics-otel"]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
|
||||
fp16kernels = ["lancedb/fp16kernels"]
|
||||
remote = ["lancedb/remote"]
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
MeterProvider,
|
||||
type MetricData,
|
||||
MetricReader,
|
||||
} from "@opentelemetry/sdk-metrics";
|
||||
import * as tmp from "tmp";
|
||||
import { connect, instrumentLanceDbMetrics } from "../lancedb";
|
||||
// snapshotLancedbMetrics is internal plumbing (not part of the public API), so
|
||||
// it is imported from the native module rather than the package entry point.
|
||||
import { snapshotLancedbMetrics } from "../lancedb/native";
|
||||
|
||||
// The metrics recorder is process-global and installed once, so the whole
|
||||
// bridge is exercised in a single test to avoid cross-test global-state coupling.
|
||||
|
||||
// A minimal pull-based reader whose `collect()` we drive directly, invoking the
|
||||
// observable-instrument callbacks. `@opentelemetry/sdk-metrics` ships no
|
||||
// in-memory reader, so we subclass the abstract base.
|
||||
class TestMetricReader extends MetricReader {
|
||||
protected async onForceFlush(): Promise<void> {
|
||||
// no-op: collection is driven directly via collect()
|
||||
}
|
||||
protected async onShutdown(): Promise<void> {
|
||||
// no-op: nothing to release
|
||||
}
|
||||
}
|
||||
|
||||
async function metricsByName(
|
||||
reader: TestMetricReader,
|
||||
): Promise<Map<string, MetricData>> {
|
||||
const collected = await reader.collect();
|
||||
const result = new Map<string, MetricData>();
|
||||
for (const scope of collected.resourceMetrics.scopeMetrics) {
|
||||
for (const metric of scope.metrics) {
|
||||
result.set(metric.descriptor.name, metric);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
describe("OpenTelemetry metrics bridge", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
afterEach(() => tmpDir.removeCallback());
|
||||
|
||||
it("snapshot is safe to call regardless of install state", () => {
|
||||
expect(Array.isArray(snapshotLancedbMetrics())).toBe(true);
|
||||
});
|
||||
|
||||
it("exports object store metrics via observable instruments", async () => {
|
||||
const reader = new TestMetricReader();
|
||||
const provider = new MeterProvider({ readers: [reader] });
|
||||
expect(instrumentLanceDbMetrics(provider)).toBe(true);
|
||||
|
||||
// Generate object store activity on the local filesystem (scheme "file").
|
||||
const db = await connect(tmpDir.name);
|
||||
const data = Array.from({ length: 256 }, (_, i) => ({ id: i }));
|
||||
const table = await db.createTable("t", data);
|
||||
expect(await table.countRows()).toBe(256);
|
||||
|
||||
const metrics = await metricsByName(reader);
|
||||
|
||||
const requests = metrics.get("lance_object_store_requests_total");
|
||||
expect(requests).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const requestPoints = (requests!.dataPoints as any[]) ?? [];
|
||||
expect(requestPoints.length).toBeGreaterThan(0);
|
||||
for (const p of requestPoints) {
|
||||
// Labelled by `operation` and `base` (the store scheme by default).
|
||||
expect(p.attributes).toHaveProperty("base");
|
||||
expect(p.attributes).toHaveProperty("operation");
|
||||
}
|
||||
const totalRequests = requestPoints.reduce((acc, p) => acc + p.value, 0);
|
||||
expect(totalRequests).toBeGreaterThan(0);
|
||||
|
||||
// Histograms are decomposed into bucket / count / sum observable counters.
|
||||
const bucket = metrics.get(
|
||||
"lance_object_store_request_duration_seconds_bucket",
|
||||
);
|
||||
expect(bucket).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const bucketPoints = (bucket!.dataPoints as any[]) ?? [];
|
||||
expect(bucketPoints.length).toBeGreaterThan(0);
|
||||
expect(bucketPoints.every((p) => "le" in p.attributes)).toBe(true);
|
||||
// The implicit +Inf bucket must be present.
|
||||
expect(bucketPoints.some((p) => p.attributes.le === "+Inf")).toBe(true);
|
||||
|
||||
const count = metrics.get(
|
||||
"lance_object_store_request_duration_seconds_count",
|
||||
);
|
||||
expect(count).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const countPoints = (count!.dataPoints as any[]) ?? [];
|
||||
expect(countPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
|
||||
|
||||
const sum = metrics.get("lance_object_store_request_duration_seconds_sum");
|
||||
expect(sum).toBeDefined();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
|
||||
const sumPoints = (sum!.dataPoints as any[]) ?? [];
|
||||
expect(sumPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
|
||||
|
||||
// Unit handling: only `_sum` keeps the histogram's unit (seconds); `_bucket`
|
||||
// and `_count` observe cumulative counts and are unitless.
|
||||
expect(sum!.descriptor.unit).toBe("s");
|
||||
expect(bucket!.descriptor.unit).toBe("");
|
||||
expect(count!.descriptor.unit).toBe("");
|
||||
|
||||
await provider.shutdown();
|
||||
});
|
||||
});
|
||||
@@ -2992,56 +2992,6 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
|
||||
}),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("reads back the installed spec via getLsmWriteSpec", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const table = await makeTable(conn);
|
||||
await table.setUnenforcedPrimaryKey("id");
|
||||
|
||||
// Nothing installed yet.
|
||||
expect(await table.getLsmWriteSpec()).toBeUndefined();
|
||||
|
||||
// A real scalar index is needed to name it as a maintained index.
|
||||
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
|
||||
await table.createIndex("id");
|
||||
const indexName = (await table.listIndices())[0].name;
|
||||
|
||||
// Bucket spec round-trips, including maintained indexes and writer config
|
||||
// defaults. Lance writer-config keys are canonically snake_case.
|
||||
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
|
||||
const writerConfigDefaults = { durable_write: "false" };
|
||||
await table.setLsmWriteSpec({
|
||||
specType: "bucket",
|
||||
column: "id",
|
||||
numBuckets: 4,
|
||||
maintainedIndexes: [indexName],
|
||||
writerConfigDefaults,
|
||||
});
|
||||
const spec = await table.getLsmWriteSpec();
|
||||
expect(spec).toBeDefined();
|
||||
expect(spec?.specType).toBe("bucket");
|
||||
expect(spec?.column).toBe("id");
|
||||
expect(spec?.numBuckets).toBe(4);
|
||||
expect(spec?.maintainedIndexes).toEqual([indexName]);
|
||||
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
|
||||
|
||||
// After unset, undefined again.
|
||||
await table.unsetLsmWriteSpec();
|
||||
expect(await table.getLsmWriteSpec()).toBeUndefined();
|
||||
|
||||
// Identity round-trips (column recovered from the schema).
|
||||
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
|
||||
const identity = await table.getLsmWriteSpec();
|
||||
expect(identity?.specType).toBe("identity");
|
||||
expect(identity?.column).toBe("id");
|
||||
await table.unsetLsmWriteSpec();
|
||||
|
||||
// Unsharded round-trips (no routing column).
|
||||
await table.setLsmWriteSpec({ specType: "unsharded" });
|
||||
const unsharded = await table.getLsmWriteSpec();
|
||||
expect(unsharded?.specType).toBe("unsharded");
|
||||
expect(unsharded?.column).toBeFalsy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("LSM merge insert", () => {
|
||||
|
||||
@@ -20,11 +20,6 @@ import { HeaderProvider } from "./header";
|
||||
// Re-export native header provider for use with connectWithHeaderProvider
|
||||
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
||||
|
||||
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
|
||||
// underlying recorder/catalog/snapshot functions remain internal plumbing that
|
||||
// `otel.ts` consumes from the native module.
|
||||
export { instrumentLanceDbMetrics } from "./otel";
|
||||
|
||||
export {
|
||||
AddColumnsSql,
|
||||
ConnectionOptions,
|
||||
|
||||
@@ -1,137 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import {
|
||||
type Attributes,
|
||||
type MeterProvider,
|
||||
type ObservableResult,
|
||||
metrics,
|
||||
} from "@opentelemetry/api";
|
||||
|
||||
import {
|
||||
lancedbMetricsCatalog,
|
||||
registerLancedbMetricsRecorder,
|
||||
snapshotLancedbMetrics,
|
||||
} from "./native";
|
||||
|
||||
let instrumented = false;
|
||||
|
||||
/**
|
||||
* Register LanceDB metrics as OpenTelemetry observable instruments.
|
||||
*
|
||||
* Installs a process-global metrics recorder and creates one observable
|
||||
* instrument per LanceDB metric (currently object store request counts, bytes,
|
||||
* latency, errors, and throttles) on the given (or global) `MeterProvider`. The
|
||||
* configured `MetricReader` then collects them on its own schedule.
|
||||
*
|
||||
* Counters and gauges map directly to observable counters/gauges. Because
|
||||
* OpenTelemetry has no asynchronous histogram instrument, each histogram is
|
||||
* exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
|
||||
* with an `le` attribute) plus `<name>_count` and `<name>_sum`.
|
||||
*
|
||||
* Requires `@opentelemetry/api` (a dependency) and, to actually export, an
|
||||
* OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
|
||||
*
|
||||
* @param meterProvider The provider to register instruments on. Defaults to the
|
||||
* global provider from `@opentelemetry/api`.
|
||||
* @returns `true` if the recorder is installed and instruments are registered.
|
||||
* `false` if a different `metrics` recorder is already installed in this
|
||||
* process (only one global recorder is permitted), in which case a warning is
|
||||
* emitted and no instruments are created. Calling this more than once is safe;
|
||||
* instruments are created only on the first successful call.
|
||||
*/
|
||||
export function instrumentLanceDbMetrics(
|
||||
meterProvider?: MeterProvider,
|
||||
): boolean {
|
||||
if (!registerLancedbMetricsRecorder()) {
|
||||
console.warn(
|
||||
"Could not install the LanceDB metrics recorder: another `metrics` " +
|
||||
"recorder is already installed in this process. LanceDB metrics will " +
|
||||
"not be exported via OpenTelemetry.",
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (instrumented) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const provider = meterProvider ?? metrics.getMeterProvider();
|
||||
const meter = provider.getMeter("lancedb");
|
||||
|
||||
const scalarCallback = (metricName: string) => (result: ObservableResult) => {
|
||||
for (const point of snapshotLancedbMetrics()) {
|
||||
if (point.name === metricName && point.value != null) {
|
||||
result.observe(point.value, point.attributes);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const bucketCallback = (metricName: string) => (result: ObservableResult) => {
|
||||
for (const point of snapshotLancedbMetrics()) {
|
||||
if (point.name !== metricName || point.buckets == null) {
|
||||
continue;
|
||||
}
|
||||
for (const bucket of point.buckets) {
|
||||
const attributes: Attributes = {
|
||||
...point.attributes,
|
||||
le: bucket.le,
|
||||
};
|
||||
result.observe(bucket.cumulativeCount, attributes);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const fieldCallback =
|
||||
(metricName: string, field: "count" | "sum") =>
|
||||
(result: ObservableResult) => {
|
||||
for (const point of snapshotLancedbMetrics()) {
|
||||
if (point.name !== metricName) {
|
||||
continue;
|
||||
}
|
||||
const value = point[field];
|
||||
if (value != null) {
|
||||
result.observe(value, point.attributes);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
for (const desc of lancedbMetricsCatalog()) {
|
||||
const unit = desc.unit ?? "";
|
||||
if (desc.kind === "counter") {
|
||||
const counter = meter.createObservableCounter(desc.name, {
|
||||
unit,
|
||||
description: desc.description,
|
||||
});
|
||||
counter.addCallback(scalarCallback(desc.name));
|
||||
} else if (desc.kind === "gauge") {
|
||||
const gauge = meter.createObservableGauge(desc.name, {
|
||||
unit,
|
||||
description: desc.description,
|
||||
});
|
||||
gauge.addCallback(scalarCallback(desc.name));
|
||||
} else if (desc.kind === "histogram") {
|
||||
// `_bucket` and `_count` observe cumulative sample counts, not the
|
||||
// histogram's measured quantity, so they are unitless; only `_sum`
|
||||
// carries the histogram's unit.
|
||||
const bucket = meter.createObservableCounter(`${desc.name}_bucket`, {
|
||||
description: `${desc.description} (cumulative buckets)`,
|
||||
});
|
||||
bucket.addCallback(bucketCallback(desc.name));
|
||||
|
||||
const count = meter.createObservableCounter(`${desc.name}_count`, {
|
||||
description: `${desc.description} (count)`,
|
||||
});
|
||||
count.addCallback(fieldCallback(desc.name, "count"));
|
||||
|
||||
const sum = meter.createObservableCounter(`${desc.name}_sum`, {
|
||||
unit,
|
||||
description: `${desc.description} (sum)`,
|
||||
});
|
||||
sum.addCallback(fieldCallback(desc.name, "sum"));
|
||||
}
|
||||
}
|
||||
|
||||
instrumented = true;
|
||||
return true;
|
||||
}
|
||||
@@ -585,17 +585,6 @@ export abstract class Table {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
abstract unsetLsmWriteSpec(): Promise<void>;
|
||||
/**
|
||||
* Read the {@link LsmWriteSpec} currently installed on this table.
|
||||
*
|
||||
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
|
||||
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
|
||||
* The returned spec — including its `maintainedIndexes` and
|
||||
* `writerConfigDefaults` — mirrors what was passed to
|
||||
* {@link Table#setLsmWriteSpec}.
|
||||
* @returns {Promise<LsmWriteSpec | undefined>}
|
||||
*/
|
||||
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
|
||||
/**
|
||||
* Drain and close any cached MemWAL shard writers held for this table.
|
||||
*
|
||||
@@ -1102,15 +1091,6 @@ export class LocalTable extends Table {
|
||||
return await this.inner.unsetLsmWriteSpec();
|
||||
}
|
||||
|
||||
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
|
||||
// The native binding types `specType` as a plain `string`; narrow it back
|
||||
// to the public union. The Rust `From` impl only ever emits one of the
|
||||
// three valid values, so the cast is safe.
|
||||
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
|
||||
| LsmWriteSpec
|
||||
| undefined;
|
||||
}
|
||||
|
||||
async closeLsmWriters(): Promise<void> {
|
||||
return await this.inner.closeLsmWriters();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
Generated
+2
-73
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@lancedb/lancedb",
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"cpu": [
|
||||
"x64",
|
||||
"arm64"
|
||||
@@ -18,7 +18,6 @@
|
||||
"win32"
|
||||
],
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"reflect-metadata": "^0.2.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -28,7 +27,6 @@
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@napi-rs/cli": "3.7.0",
|
||||
"@opentelemetry/sdk-metrics": "^1.30.0",
|
||||
"@types/axios": "^0.14.0",
|
||||
"@types/jest": "^29.1.2",
|
||||
"@types/node": "22.7.4",
|
||||
@@ -4150,75 +4148,6 @@
|
||||
"@octokit/openapi-types": "^27.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/api": {
|
||||
"version": "1.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz",
|
||||
"integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/core": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz",
|
||||
"integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/resources": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz",
|
||||
"integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/semantic-conventions": "1.28.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.0.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/sdk-metrics": {
|
||||
"version": "1.30.1",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.30.1.tgz",
|
||||
"integrity": "sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@opentelemetry/core": "1.30.1",
|
||||
"@opentelemetry/resources": "1.30.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@opentelemetry/api": ">=1.3.0 <1.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@opentelemetry/semantic-conventions": {
|
||||
"version": "1.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz",
|
||||
"integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/@protobufjs/aspromise": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
|
||||
|
||||
+1
-3
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.32.0-beta.0",
|
||||
"version": "0.31.0-beta.6",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
@@ -44,7 +44,6 @@
|
||||
"@biomejs/biome": "^1.7.3",
|
||||
"@jest/globals": "^29.7.0",
|
||||
"@napi-rs/cli": "3.7.0",
|
||||
"@opentelemetry/sdk-metrics": "^1.30.0",
|
||||
"@types/axios": "^0.14.0",
|
||||
"@types/jest": "^29.1.2",
|
||||
"@types/node": "22.7.4",
|
||||
@@ -93,7 +92,6 @@
|
||||
"version": "napi version"
|
||||
},
|
||||
"dependencies": {
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"reflect-metadata": "^0.2.2"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
|
||||
Generated
-53
@@ -8,9 +8,6 @@ importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
'@opentelemetry/api':
|
||||
specifier: ^1.9.0
|
||||
version: 1.9.1
|
||||
apache-arrow:
|
||||
specifier: '>=15.0.0 <=18.1.0'
|
||||
version: 18.1.0
|
||||
@@ -36,9 +33,6 @@ importers:
|
||||
'@napi-rs/cli':
|
||||
specifier: 3.7.0
|
||||
version: 3.7.0(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@22.7.4)
|
||||
'@opentelemetry/sdk-metrics':
|
||||
specifier: ^1.30.0
|
||||
version: 1.30.1(@opentelemetry/api@1.9.1)
|
||||
'@types/axios':
|
||||
specifier: ^0.14.0
|
||||
version: 0.14.4
|
||||
@@ -1313,32 +1307,6 @@ packages:
|
||||
'@octokit/types@16.0.0':
|
||||
resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==}
|
||||
|
||||
'@opentelemetry/api@1.9.1':
|
||||
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
|
||||
engines: {node: '>=8.0.0'}
|
||||
|
||||
'@opentelemetry/core@1.30.1':
|
||||
resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==}
|
||||
engines: {node: '>=14'}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/resources@1.30.1':
|
||||
resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==}
|
||||
engines: {node: '>=14'}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.0.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/sdk-metrics@1.30.1':
|
||||
resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==}
|
||||
engines: {node: '>=14'}
|
||||
peerDependencies:
|
||||
'@opentelemetry/api': '>=1.3.0 <1.10.0'
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.28.0':
|
||||
resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==}
|
||||
engines: {node: '>=14'}
|
||||
|
||||
'@protobufjs/aspromise@1.1.2':
|
||||
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
|
||||
|
||||
@@ -4957,27 +4925,6 @@ snapshots:
|
||||
dependencies:
|
||||
'@octokit/openapi-types': 27.0.0
|
||||
|
||||
'@opentelemetry/api@1.9.1': {}
|
||||
|
||||
'@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/semantic-conventions': 1.28.0
|
||||
|
||||
'@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/semantic-conventions': 1.28.0
|
||||
|
||||
'@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.1)':
|
||||
dependencies:
|
||||
'@opentelemetry/api': 1.9.1
|
||||
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
|
||||
'@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1)
|
||||
|
||||
'@opentelemetry/semantic-conventions@1.28.0': {}
|
||||
|
||||
'@protobufjs/aspromise@1.1.2':
|
||||
optional: true
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ mod header;
|
||||
mod index;
|
||||
mod iterator;
|
||||
pub mod merge;
|
||||
pub mod otel;
|
||||
pub mod permutation;
|
||||
mod query;
|
||||
pub mod remote;
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Node.js bindings over [`lancedb::metrics_otel`].
|
||||
//!
|
||||
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
|
||||
//! core crate; this module only converts the core snapshot types into napi
|
||||
//! objects and exposes the three entry points to JavaScript, where
|
||||
//! `lancedb/otel.ts` bridges them into the user's OpenTelemetry `MeterProvider`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use lancedb::metrics_otel::{MetricPoint as CoreMetricPoint, MetricValue};
|
||||
use napi_derive::napi;
|
||||
|
||||
/// One cumulative histogram bucket: all samples with value `<= le`.
|
||||
#[napi(object)]
|
||||
pub struct MetricBucket {
|
||||
/// The inclusive upper bound of the bucket, or `"+Inf"` for the final bucket.
|
||||
pub le: String,
|
||||
/// Cumulative number of samples less than or equal to `le`.
|
||||
pub cumulative_count: f64,
|
||||
}
|
||||
|
||||
/// One aggregated metric data point. For counters and gauges only `value` is
|
||||
/// set; for histograms `buckets` (cumulative `le` counts), `count`, and `sum`
|
||||
/// are set.
|
||||
#[napi(object)]
|
||||
pub struct MetricPoint {
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub attributes: HashMap<String, String>,
|
||||
pub value: Option<f64>,
|
||||
pub buckets: Option<Vec<MetricBucket>>,
|
||||
pub count: Option<f64>,
|
||||
pub sum: Option<f64>,
|
||||
}
|
||||
|
||||
impl From<CoreMetricPoint> for MetricPoint {
|
||||
fn from(point: CoreMetricPoint) -> Self {
|
||||
let kind = point.kind.as_str().to_string();
|
||||
let (value, buckets, count, sum) = match point.value {
|
||||
MetricValue::Scalar(v) => (Some(v), None, None, None),
|
||||
MetricValue::Histogram {
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
} => (
|
||||
None,
|
||||
Some(
|
||||
buckets
|
||||
.into_iter()
|
||||
// Counts stay well within the f64-exact integer range
|
||||
// (2^53), so this cast is lossless in practice and keeps
|
||||
// the values plain JS numbers for OpenTelemetry.
|
||||
.map(|(le, cumulative_count)| MetricBucket {
|
||||
le,
|
||||
cumulative_count: cumulative_count as f64,
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
Some(count as f64),
|
||||
Some(sum),
|
||||
),
|
||||
};
|
||||
Self {
|
||||
name: point.name,
|
||||
kind,
|
||||
attributes: point.attributes,
|
||||
value,
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A described metric, used by the JavaScript layer to create instruments up front.
|
||||
#[napi(object)]
|
||||
pub struct MetricDescription {
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub unit: Option<String>,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
|
||||
///
|
||||
/// Returns `true` if the recorder is installed (now or previously). Returns
|
||||
/// `false` if a *different* recorder is already installed — `metrics` allows
|
||||
/// only one global recorder per process, so LanceDB cannot coexist with another.
|
||||
#[napi]
|
||||
pub fn register_lancedb_metrics_recorder() -> bool {
|
||||
lancedb::metrics_otel::register_metrics_recorder()
|
||||
}
|
||||
|
||||
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
|
||||
#[napi]
|
||||
pub fn lancedb_metrics_catalog() -> Vec<MetricDescription> {
|
||||
lancedb::metrics_otel::metrics_catalog()
|
||||
.into_iter()
|
||||
.map(|desc| MetricDescription {
|
||||
name: desc.name,
|
||||
kind: desc.kind.as_str().to_string(),
|
||||
unit: desc.unit,
|
||||
description: desc.description,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
|
||||
/// is installed.
|
||||
#[napi]
|
||||
pub fn snapshot_lancedb_metrics() -> Vec<MetricPoint> {
|
||||
lancedb::metrics_otel::snapshot_metrics()
|
||||
.into_iter()
|
||||
.map(MetricPoint::from)
|
||||
.collect()
|
||||
}
|
||||
@@ -411,16 +411,6 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
|
||||
let spec = self
|
||||
.inner_ref()?
|
||||
.get_lsm_write_spec()
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(spec.map(LsmWriteSpec::from))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.close_lsm_writers().await.default_error()
|
||||
@@ -738,47 +728,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
|
||||
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
|
||||
use lancedb::table::LsmWriteSpec as Native;
|
||||
match spec {
|
||||
Native::Bucket {
|
||||
column,
|
||||
num_buckets,
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => Self {
|
||||
spec_type: "bucket".to_string(),
|
||||
column: Some(column),
|
||||
num_buckets: Some(num_buckets),
|
||||
maintained_indexes: Some(maintained_indexes),
|
||||
writer_config_defaults: Some(writer_config_defaults),
|
||||
},
|
||||
Native::Identity {
|
||||
column,
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => Self {
|
||||
spec_type: "identity".to_string(),
|
||||
column: Some(column),
|
||||
num_buckets: None,
|
||||
maintained_indexes: Some(maintained_indexes),
|
||||
writer_config_defaults: Some(writer_config_defaults),
|
||||
},
|
||||
Native::Unsharded {
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => Self {
|
||||
spec_type: "unsharded".to_string(),
|
||||
column: None,
|
||||
num_buckets: None,
|
||||
maintained_indexes: Some(maintained_indexes),
|
||||
writer_config_defaults: Some(writer_config_defaults),
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Statistics about a compaction operation.
|
||||
#[napi(object)]
|
||||
#[derive(Clone, Debug)]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.35.0-beta.1"
|
||||
current_version = "0.34.0-beta.6"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.35.0-beta.1"
|
||||
version = "0.34.0-beta.6"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
@@ -47,6 +47,6 @@ pyo3-build-config = { version = "0.28", features = [
|
||||
] }
|
||||
|
||||
[features]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
|
||||
fp16kernels = ["lancedb/fp16kernels"]
|
||||
remote = ["lancedb/remote"]
|
||||
|
||||
@@ -47,10 +47,6 @@ repository = "https://github.com/lancedb/lancedb"
|
||||
pylance = [
|
||||
"pylance>=5.0.0b5",
|
||||
]
|
||||
# A library only needs the OpenTelemetry API; the application supplies and
|
||||
# configures the SDK (the actual exporter/reader). See
|
||||
# https://opentelemetry.io/docs/languages/python/instrumentation/
|
||||
otel = ["opentelemetry-api"]
|
||||
tests = [
|
||||
"aiohttp>=3.9.0",
|
||||
"boto3>=1.28.57",
|
||||
@@ -65,7 +61,6 @@ tests = [
|
||||
"pylance>=5.0.0b5",
|
||||
"requests>=2.31.0",
|
||||
"datafusion>=52,<53",
|
||||
"opentelemetry-sdk>=1.30.0",
|
||||
]
|
||||
dev = [
|
||||
"ruff>=0.3.0",
|
||||
|
||||
@@ -149,14 +149,8 @@ def connect(
|
||||
|
||||
For object storage, use a URI prefix:
|
||||
|
||||
>>> db = lancedb.connect( # doctest: +SKIP
|
||||
... "s3://my-bucket/lancedb",
|
||||
... storage_options={
|
||||
... "aws_access_key_id": "***",
|
||||
... "aws_secret_access_key": "***",
|
||||
... "aws_region": "us-east-1",
|
||||
... },
|
||||
... )
|
||||
>>> db = lancedb.connect("s3://my-bucket/lancedb",
|
||||
... storage_options={"aws_access_key_id": "***"})
|
||||
|
||||
For tests and temporary data, use an in-memory database:
|
||||
|
||||
|
||||
@@ -30,25 +30,6 @@ IvfHnswPq: type[HnswPq] = HnswPq
|
||||
IvfHnswSq: type[HnswSq] = HnswSq
|
||||
IvfHnswFlat: type[HnswFlat] = HnswFlat
|
||||
|
||||
class MetricPoint:
|
||||
name: str
|
||||
kind: str
|
||||
attributes: Dict[str, str]
|
||||
value: Optional[float]
|
||||
buckets: Optional[List[Tuple[str, int]]]
|
||||
count: Optional[int]
|
||||
sum: Optional[float]
|
||||
|
||||
class MetricDescription:
|
||||
name: str
|
||||
kind: str
|
||||
unit: Optional[str]
|
||||
description: str
|
||||
|
||||
def register_lancedb_metrics_recorder() -> bool: ...
|
||||
def lancedb_metrics_catalog() -> List[MetricDescription]: ...
|
||||
def snapshot_lancedb_metrics() -> List[MetricPoint]: ...
|
||||
|
||||
class PyExpr:
|
||||
"""A type-safe DataFusion expression node (Rust-side handle)."""
|
||||
|
||||
@@ -248,7 +229,6 @@ class Table:
|
||||
async def set_unenforced_primary_key(self, columns: List[str]) -> None: ...
|
||||
async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
|
||||
async def unset_lsm_write_spec(self) -> None: ...
|
||||
async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
|
||||
async def close_lsm_writers(self) -> None: ...
|
||||
@property
|
||||
def tags(self) -> Tags: ...
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Union
|
||||
from typing import TYPE_CHECKING, List, Optional, Sequence, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
@@ -56,16 +56,6 @@ class OllamaEmbeddings(TextEmbeddingFunction):
|
||||
embeddings = self._compute_embedding(texts)
|
||||
return list(embeddings)
|
||||
|
||||
def __getstate__(self) -> dict[str, Any]:
|
||||
state = super().__getstate__()
|
||||
state["__dict__"] = {
|
||||
k: v for k, v in state["__dict__"].items() if k != "_ollama_client"
|
||||
}
|
||||
return state
|
||||
|
||||
def __setstate__(self, state: dict[str, Any]) -> None:
|
||||
super().__setstate__(state)
|
||||
|
||||
@cached_property
|
||||
def _ollama_client(self) -> "ollama.Client":
|
||||
ollama = attempt_import_or_raise("ollama")
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Bridge LanceDB's internal metrics into OpenTelemetry.
|
||||
|
||||
LanceDB (through Lance core) publishes metrics (currently object store request
|
||||
counts, bytes, latency, errors, and throttles) through the Rust ``metrics``
|
||||
facade. This module installs a process-global recorder that aggregates them and
|
||||
registers OpenTelemetry observable instruments that report the aggregated values
|
||||
into the user's ``MeterProvider``.
|
||||
|
||||
The bridge is generic: every metric LanceDB describes is surfaced automatically,
|
||||
with no per-metric Python code. Histograms have no asynchronous OpenTelemetry
|
||||
instrument, so each is exported Prometheus-style as cumulative ``le`` buckets
|
||||
plus ``_count`` and ``_sum`` observable counters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from ._lancedb import (
|
||||
lancedb_metrics_catalog,
|
||||
register_lancedb_metrics_recorder,
|
||||
snapshot_lancedb_metrics,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.metrics import MeterProvider
|
||||
|
||||
_INSTRUMENTED = False
|
||||
|
||||
|
||||
def instrument_lancedb_metrics(
|
||||
meter_provider: Optional["MeterProvider"] = None,
|
||||
) -> bool:
|
||||
"""Register LanceDB metrics as OpenTelemetry observable instruments.
|
||||
|
||||
Installs a process-global metrics recorder and creates one observable
|
||||
instrument per LanceDB metric on the given (or global) ``MeterProvider``. The
|
||||
user's configured ``MetricReader`` then collects them on its own schedule.
|
||||
|
||||
Counters and gauges map directly to observable counters/gauges. Each
|
||||
histogram is exported as cumulative ``le`` bucket counts (``<name>_bucket``,
|
||||
with an ``le`` attribute) plus ``<name>_count`` and ``<name>_sum``.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
meter_provider : opentelemetry.metrics.MeterProvider, optional
|
||||
The provider to register instruments on. Defaults to the global provider
|
||||
from ``opentelemetry.metrics.get_meter_provider()``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
``True`` if the recorder is installed and instruments are registered.
|
||||
``False`` if a different ``metrics`` recorder is already installed in
|
||||
this process (``metrics`` permits only one global recorder), in which
|
||||
case a warning is emitted and no instruments are created.
|
||||
|
||||
Notes
|
||||
-----
|
||||
Requires the OpenTelemetry API (``pip install lancedb[otel]``) and, to
|
||||
actually export, an OpenTelemetry SDK (``pip install opentelemetry-sdk``)
|
||||
configured by the application. Calling this more than once is safe;
|
||||
instruments are created only on the first successful call.
|
||||
"""
|
||||
global _INSTRUMENTED
|
||||
|
||||
try:
|
||||
from opentelemetry.metrics import Observation, get_meter_provider
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"instrument_lancedb_metrics requires the OpenTelemetry API/SDK. "
|
||||
"Install it with `pip install lancedb[otel]` or "
|
||||
"`pip install opentelemetry-sdk`."
|
||||
) from exc
|
||||
|
||||
if not register_lancedb_metrics_recorder():
|
||||
warnings.warn(
|
||||
"Could not install the LanceDB metrics recorder: another `metrics` "
|
||||
"recorder is already installed in this process. LanceDB metrics will "
|
||||
"not be exported via OpenTelemetry.",
|
||||
stacklevel=2,
|
||||
)
|
||||
return False
|
||||
|
||||
if _INSTRUMENTED:
|
||||
return True
|
||||
|
||||
provider = meter_provider or get_meter_provider()
|
||||
meter = provider.get_meter("lancedb")
|
||||
|
||||
def scalar_callback(metric_name: str):
|
||||
def callback(_options):
|
||||
return [
|
||||
Observation(point.value, point.attributes)
|
||||
for point in snapshot_lancedb_metrics()
|
||||
if point.name == metric_name and point.value is not None
|
||||
]
|
||||
|
||||
return callback
|
||||
|
||||
def bucket_callback(metric_name: str):
|
||||
def callback(_options):
|
||||
observations = []
|
||||
for point in snapshot_lancedb_metrics():
|
||||
if point.name != metric_name or point.buckets is None:
|
||||
continue
|
||||
for le, cumulative in point.buckets:
|
||||
attributes = dict(point.attributes)
|
||||
attributes["le"] = le
|
||||
observations.append(Observation(cumulative, attributes))
|
||||
return observations
|
||||
|
||||
return callback
|
||||
|
||||
def field_callback(metric_name: str, field: str):
|
||||
def callback(_options):
|
||||
observations = []
|
||||
for point in snapshot_lancedb_metrics():
|
||||
if point.name != metric_name:
|
||||
continue
|
||||
value = getattr(point, field)
|
||||
if value is not None:
|
||||
observations.append(Observation(value, point.attributes))
|
||||
return observations
|
||||
|
||||
return callback
|
||||
|
||||
for desc in lancedb_metrics_catalog():
|
||||
unit = desc.unit or ""
|
||||
if desc.kind == "counter":
|
||||
meter.create_observable_counter(
|
||||
desc.name,
|
||||
callbacks=[scalar_callback(desc.name)],
|
||||
unit=unit,
|
||||
description=desc.description,
|
||||
)
|
||||
elif desc.kind == "gauge":
|
||||
meter.create_observable_gauge(
|
||||
desc.name,
|
||||
callbacks=[scalar_callback(desc.name)],
|
||||
unit=unit,
|
||||
description=desc.description,
|
||||
)
|
||||
elif desc.kind == "histogram":
|
||||
# `_bucket` and `_count` observe cumulative sample counts, not the
|
||||
# histogram's measured quantity, so they are unitless; only `_sum`
|
||||
# carries the histogram's unit.
|
||||
meter.create_observable_counter(
|
||||
f"{desc.name}_bucket",
|
||||
callbacks=[bucket_callback(desc.name)],
|
||||
description=f"{desc.description} (cumulative buckets)",
|
||||
)
|
||||
meter.create_observable_counter(
|
||||
f"{desc.name}_count",
|
||||
callbacks=[field_callback(desc.name, "count")],
|
||||
description=f"{desc.description} (count)",
|
||||
)
|
||||
meter.create_observable_counter(
|
||||
f"{desc.name}_sum",
|
||||
callbacks=[field_callback(desc.name, "sum")],
|
||||
unit=unit,
|
||||
description=f"{desc.description} (sum)",
|
||||
)
|
||||
|
||||
_INSTRUMENTED = True
|
||||
return True
|
||||
@@ -11,7 +11,7 @@ import pyarrow as pa
|
||||
from ._lancedb import async_permutation_builder, PermutationReader
|
||||
from .table import LanceTable, Table
|
||||
from .background_loop import LOOP
|
||||
from .util import batch_to_tensor, batch_to_tensor_dict, batch_to_tensor_rows
|
||||
from .util import batch_to_tensor, batch_to_tensor_rows
|
||||
from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -946,7 +946,6 @@ class Permutation:
|
||||
"pandas",
|
||||
"arrow",
|
||||
"torch",
|
||||
"torch_row",
|
||||
"torch_col",
|
||||
"polars",
|
||||
],
|
||||
@@ -962,19 +961,15 @@ class Permutation:
|
||||
- "python_col" - the batch will be a dict of lists (one entry per column)
|
||||
- "pandas" - the batch will be a pandas DataFrame
|
||||
- "arrow" - the batch will be a pyarrow RecordBatch
|
||||
- "torch" - the batch will be a list of per-row dicts mapping column
|
||||
name to a 0-D torch tensor. Works with the default
|
||||
``torch.utils.data.DataLoader`` collate, which stacks the per-row
|
||||
dicts back into a dict of batched tensors.
|
||||
- "torch_row" - the batch will be a list of tensors, one per row
|
||||
- "torch" - the batch will be a list of tensors, one per row
|
||||
- "torch_col" - the batch will be a 2D torch tensor (first dim indexes columns)
|
||||
- "polars" - the batch will be a polars DataFrame
|
||||
|
||||
Conversion may or may not involve a data copy. Lance uses Arrow internally
|
||||
and so it is able to zero-copy to the arrow and polars formats.
|
||||
|
||||
Conversion to torch and torch_col will be zero-copy but will only support a
|
||||
subset of data types (numeric types).
|
||||
Conversion to torch_col will be zero-copy but will only support a subset of data
|
||||
types (numeric types).
|
||||
|
||||
Conversion to numpy and/or pandas will typically be zero-copy for numeric
|
||||
types. Conversion of strings, lists, and structs will require creating python
|
||||
@@ -995,8 +990,6 @@ class Permutation:
|
||||
elif format == "arrow":
|
||||
return self.with_transform(Transforms.arrow2arrow)
|
||||
elif format == "torch":
|
||||
return self.with_transform(batch_to_tensor_dict)
|
||||
elif format == "torch_row":
|
||||
return self.with_transform(batch_to_tensor_rows)
|
||||
elif format == "torch_col":
|
||||
return self.with_transform(batch_to_tensor)
|
||||
|
||||
@@ -912,10 +912,6 @@ class RemoteTable(Table):
|
||||
"""Not supported on LanceDB Cloud."""
|
||||
return LOOP.run(self._table.unset_lsm_write_spec())
|
||||
|
||||
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
|
||||
"""Read the installed LsmWriteSpec, or ``None``."""
|
||||
return LOOP.run(self._table.get_lsm_write_spec())
|
||||
|
||||
def close_lsm_writers(self) -> None:
|
||||
"""No-op on LanceDB Cloud (no local shard writers)."""
|
||||
return LOOP.run(self._table.close_lsm_writers())
|
||||
|
||||
@@ -3759,11 +3759,6 @@ class LanceTable(Table):
|
||||
[`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec]."""
|
||||
return LOOP.run(self._table.unset_lsm_write_spec())
|
||||
|
||||
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
|
||||
"""Read the installed LsmWriteSpec, or ``None``. See
|
||||
[`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
|
||||
return LOOP.run(self._table.get_lsm_write_spec())
|
||||
|
||||
def close_lsm_writers(self) -> None:
|
||||
"""Close cached MemWAL shard writers. See
|
||||
[`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
|
||||
@@ -4422,17 +4417,6 @@ class AsyncTable:
|
||||
"""
|
||||
await self._inner.unset_lsm_write_spec()
|
||||
|
||||
async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
|
||||
"""Read the LsmWriteSpec currently installed on this table.
|
||||
|
||||
Returns ``None`` when the MemWAL LSM write path is not enabled (no
|
||||
spec has been set, or it was removed with `unset_lsm_write_spec`).
|
||||
The returned spec — including its ``maintained_indexes`` and
|
||||
``writer_config_defaults`` — mirrors what was passed to
|
||||
`set_lsm_write_spec`.
|
||||
"""
|
||||
return await self._inner.get_lsm_write_spec()
|
||||
|
||||
async def close_lsm_writers(self) -> None:
|
||||
"""Drain and close any cached MemWAL shard writers for this table.
|
||||
|
||||
|
||||
@@ -177,10 +177,7 @@ def flatten_columns(tbl: pa.Table, flatten: Optional[Union[int, bool]] = None):
|
||||
continue
|
||||
else:
|
||||
break
|
||||
# `bool` is a subclass of `int`, so guard against it explicitly: `flatten=False`
|
||||
# (and `None`) must mean "do not flatten" rather than falling into the integer
|
||||
# branch and raising on the `flatten <= 0` check.
|
||||
elif isinstance(flatten, int) and not isinstance(flatten, bool):
|
||||
elif isinstance(flatten, int):
|
||||
if flatten <= 0:
|
||||
raise ValueError(
|
||||
"Please specify a positive integer for flatten or the boolean "
|
||||
@@ -518,38 +515,3 @@ def batch_to_tensor_rows(batch: pa.RecordBatch):
|
||||
stacked = torch.tensor(numpy.column_stack(columns))
|
||||
rows = list(stacked.unbind(dim=0))
|
||||
return rows
|
||||
|
||||
|
||||
def batch_to_tensor_dict(batch: pa.RecordBatch):
|
||||
"""
|
||||
Convert a PyArrow RecordBatch to a list of per-row dicts of PyTorch Tensors.
|
||||
|
||||
Each column is first converted to a 1-D tensor (zero-copy via DLPack), then
|
||||
sliced per row. The result is a list whose length is ``batch.num_rows`` and
|
||||
whose items are dicts keyed by column name. This shape composes directly
|
||||
with PyTorch's default ``DataLoader`` collate, which stacks the per-row
|
||||
dicts back into a dict of batched tensors.
|
||||
|
||||
Fails if torch is not installed.
|
||||
Fails if a column's data type is not supported by PyTorch.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
batch : pa.RecordBatch
|
||||
The record batch to convert.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[dict[str, torch.Tensor]]
|
||||
One per-row dict per row in the batch. Each dict maps column name to a
|
||||
0-D tensor view into the column.
|
||||
"""
|
||||
torch = attempt_import_or_raise("torch", "torch")
|
||||
tensors = {
|
||||
name: torch.from_dlpack(col)
|
||||
for name, col in zip(batch.schema.names, batch.columns)
|
||||
}
|
||||
return [
|
||||
{name: tensor[i] for name, tensor in tensors.items()}
|
||||
for i in range(batch.num_rows)
|
||||
]
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import os
|
||||
import pickle
|
||||
from typing import List, Optional, Union
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -592,22 +591,6 @@ def test_openai_no_retry_on_401(mock_sleep):
|
||||
assert mock_sleep.call_count == 0
|
||||
|
||||
|
||||
def test_ollama_embeddings_pickle():
|
||||
"""OllamaEmbeddings must pickle even after the cached client is created."""
|
||||
registry = get_registry()
|
||||
model = registry.get("ollama").create(name="nomic-embed-text")
|
||||
|
||||
# Simulate accessing the cached client, which stores it on the instance.
|
||||
model.__dict__["_ollama_client"] = MagicMock()
|
||||
|
||||
pickled = pickle.dumps(model)
|
||||
restored = pickle.loads(pickled)
|
||||
|
||||
assert restored.name == "nomic-embed-text"
|
||||
assert restored.host == "http://localhost:11434"
|
||||
assert "_ollama_client" not in restored.__dict__
|
||||
|
||||
|
||||
def test_url_retrieve_downloads_image():
|
||||
"""
|
||||
Embedding functions like open-clip, siglip, and jinaai use url_retrieve()
|
||||
|
||||
@@ -11,7 +11,6 @@ import lancedb
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from lancedb._lancedb import LsmWriteSpec
|
||||
from lancedb.index import BTree
|
||||
|
||||
SCHEMA = pa.schema(
|
||||
[
|
||||
@@ -137,76 +136,3 @@ def test_lsm_write_spec_identity_and_writer_config_defaults():
|
||||
s = s.with_writer_config_defaults({"durable_write": "false"})
|
||||
assert s.writer_config_defaults == {"durable_write": "false"}
|
||||
assert "durable_write" in repr(s)
|
||||
|
||||
|
||||
def test_get_lsm_write_spec(tmp_path):
|
||||
_db, table = _make_table(tmp_path)
|
||||
table.set_unenforced_primary_key("id")
|
||||
|
||||
# None when nothing is installed.
|
||||
assert table.get_lsm_write_spec() is None
|
||||
|
||||
# A real scalar index is needed to name it as a maintained index.
|
||||
table.create_index("id", config=BTree())
|
||||
idx_name = table.list_indices()[0].name
|
||||
|
||||
# Bucket spec round-trips, including maintained indexes and writer config
|
||||
# defaults.
|
||||
table.set_lsm_write_spec(
|
||||
LsmWriteSpec.bucket("id", 4)
|
||||
.with_maintained_indexes([idx_name])
|
||||
.with_writer_config_defaults({"durable_write": "false"})
|
||||
)
|
||||
spec = table.get_lsm_write_spec()
|
||||
assert spec is not None
|
||||
assert spec.spec_type == "bucket"
|
||||
assert spec.column == "id"
|
||||
assert spec.num_buckets == 4
|
||||
assert spec.maintained_indexes == [idx_name]
|
||||
assert spec.writer_config_defaults == {"durable_write": "false"}
|
||||
|
||||
# After unset, None again.
|
||||
table.unset_lsm_write_spec()
|
||||
assert table.get_lsm_write_spec() is None
|
||||
|
||||
# Identity round-trips (column recovered from the schema).
|
||||
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
|
||||
spec = table.get_lsm_write_spec()
|
||||
assert spec.spec_type == "identity"
|
||||
assert spec.column == "id"
|
||||
table.unset_lsm_write_spec()
|
||||
|
||||
# Unsharded round-trips (no routing column).
|
||||
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
|
||||
spec = table.get_lsm_write_spec()
|
||||
assert spec.spec_type == "unsharded"
|
||||
assert spec.column is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_lsm_write_spec(tmp_path):
|
||||
db = await lancedb.connect_async(
|
||||
tmp_path, read_consistency_interval=timedelta(seconds=0)
|
||||
)
|
||||
table = await db.create_table(
|
||||
"t",
|
||||
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
|
||||
)
|
||||
|
||||
assert await table.get_lsm_write_spec() is None
|
||||
|
||||
# A real scalar index is needed to name it as a maintained index.
|
||||
await table.create_index("id", config=BTree())
|
||||
idx_name = (await table.list_indices())[0].name
|
||||
|
||||
await table.set_lsm_write_spec(
|
||||
LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
|
||||
)
|
||||
spec = await table.get_lsm_write_spec()
|
||||
assert spec is not None
|
||||
assert spec.spec_type == "bucket"
|
||||
assert spec.column == "id"
|
||||
assert spec.num_buckets == 8
|
||||
assert spec.maintained_indexes == [idx_name]
|
||||
await table.unset_lsm_write_spec()
|
||||
assert await table.get_lsm_write_spec() is None
|
||||
|
||||
@@ -935,41 +935,14 @@ def test_transform_fn(mem_db):
|
||||
try:
|
||||
import torch
|
||||
|
||||
# "torch" returns a list of per-row dicts. Default DataLoader collate
|
||||
# stacks the per-row dicts back into a dict of batched tensors.
|
||||
torch_perm = permutation.with_format("torch")
|
||||
torch_batch = list(torch_perm.iter(10, skip_last_batch=False))[0]
|
||||
assert isinstance(torch_batch, list)
|
||||
assert len(torch_batch) == 10
|
||||
assert isinstance(torch_batch[0], dict)
|
||||
assert set(torch_batch[0].keys()) == {"id", "value"}
|
||||
assert isinstance(torch_batch[0]["id"], torch.Tensor)
|
||||
assert torch_batch[0]["id"].dtype == torch.int64
|
||||
|
||||
rows = torch_perm.__getitems__([0, 1, 2])
|
||||
assert isinstance(rows, list)
|
||||
assert len(rows) == 3
|
||||
assert isinstance(rows[0], dict)
|
||||
assert set(rows[0].keys()) == {"id", "value"}
|
||||
assert isinstance(rows[0]["id"], torch.Tensor)
|
||||
|
||||
# "torch_row" returns a list of tensors, one per row.
|
||||
torch_rows = list(
|
||||
permutation.with_format("torch_row").iter(10, skip_last_batch=False)
|
||||
torch_result = list(
|
||||
permutation.with_format("torch").iter(10, skip_last_batch=False)
|
||||
)[0]
|
||||
assert isinstance(torch_rows, list)
|
||||
assert len(torch_rows) == 10
|
||||
assert isinstance(torch_rows[0], torch.Tensor)
|
||||
assert torch_rows[0].shape == (2,)
|
||||
assert torch_rows[0].dtype == torch.int64
|
||||
|
||||
# "torch_col" stacks columns into a single 2D tensor.
|
||||
torch_col = list(
|
||||
permutation.with_format("torch_col").iter(10, skip_last_batch=False)
|
||||
)[0]
|
||||
assert isinstance(torch_col, torch.Tensor)
|
||||
assert torch_col.shape == (2, 10)
|
||||
assert torch_col.dtype == torch.int64
|
||||
assert isinstance(torch_result, list)
|
||||
assert len(torch_result) == 10
|
||||
assert isinstance(torch_result[0], torch.Tensor)
|
||||
assert torch_result[0].shape == (2,)
|
||||
assert torch_result[0].dtype == torch.int64
|
||||
except ImportError:
|
||||
# Skip check if torch is not installed
|
||||
pass
|
||||
|
||||
@@ -7,8 +7,7 @@ Tests for S3 bucket names containing dots.
|
||||
Related issue: https://github.com/lancedb/lancedb/issues/1898
|
||||
|
||||
These tests validate the early error checking for S3 bucket names with dots.
|
||||
When validation succeeds, eager namespace initialization may still fail later
|
||||
because these tests intentionally do not provide real S3 credentials.
|
||||
No actual S3 connection is made - validation happens before connection.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -21,13 +20,6 @@ BUCKET_WITH_DOTS_AND_AWS_REGION = ("s3://my.bucket.name", {"aws_region": "us-eas
|
||||
BUCKET_WITHOUT_DOTS = "s3://my-bucket/path"
|
||||
|
||||
|
||||
def assert_not_rejected_for_bucket_dots(connect):
|
||||
try:
|
||||
connect()
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
|
||||
|
||||
class TestS3BucketWithDotsSync:
|
||||
"""Tests for connect()."""
|
||||
|
||||
@@ -35,22 +27,19 @@ class TestS3BucketWithDotsSync:
|
||||
with pytest.raises(ValueError, match="contains dots"):
|
||||
lancedb.connect(BUCKET_WITH_DOTS)
|
||||
|
||||
def test_bucket_with_dots_and_region_is_not_rejected(self):
|
||||
def test_bucket_with_dots_and_region_passes(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_REGION
|
||||
assert_not_rejected_for_bucket_dots(
|
||||
lambda: lancedb.connect(uri, storage_options=opts)
|
||||
)
|
||||
db = lancedb.connect(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
|
||||
def test_bucket_with_dots_and_aws_region_is_not_rejected(self):
|
||||
def test_bucket_with_dots_and_aws_region_passes(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
|
||||
assert_not_rejected_for_bucket_dots(
|
||||
lambda: lancedb.connect(uri, storage_options=opts)
|
||||
)
|
||||
db = lancedb.connect(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
|
||||
def test_bucket_without_dots_is_not_rejected(self):
|
||||
assert_not_rejected_for_bucket_dots(
|
||||
lambda: lancedb.connect(BUCKET_WITHOUT_DOTS)
|
||||
)
|
||||
def test_bucket_without_dots_passes(self):
|
||||
db = lancedb.connect(BUCKET_WITHOUT_DOTS)
|
||||
assert db is not None
|
||||
|
||||
|
||||
class TestS3BucketWithDotsAsync:
|
||||
@@ -62,24 +51,18 @@ class TestS3BucketWithDotsAsync:
|
||||
await lancedb.connect_async(BUCKET_WITH_DOTS)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bucket_with_dots_and_region_is_not_rejected(self):
|
||||
async def test_bucket_with_dots_and_region_passes(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_REGION
|
||||
try:
|
||||
await lancedb.connect_async(uri, storage_options=opts)
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
db = await lancedb.connect_async(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bucket_with_dots_and_aws_region_is_not_rejected(self):
|
||||
async def test_bucket_with_dots_and_aws_region_passes(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
|
||||
try:
|
||||
await lancedb.connect_async(uri, storage_options=opts)
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
db = await lancedb.connect_async(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bucket_without_dots_is_not_rejected(self):
|
||||
try:
|
||||
await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
async def test_bucket_without_dots_passes(self):
|
||||
db = await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
|
||||
assert db is not None
|
||||
|
||||
@@ -148,47 +148,15 @@ def test_permutation_dataloader(mem_db):
|
||||
for batch in dataloader:
|
||||
assert batch["a"].size(0) == 10
|
||||
|
||||
# "torch" produces a list of per-row dicts per batch. The default
|
||||
# DataLoader collate stacks the per-row dicts back into a batched dict.
|
||||
torch_perm = permutation.with_format("torch")
|
||||
batch = next(torch_perm.iter(10, skip_last_batch=False))
|
||||
assert isinstance(batch, list)
|
||||
assert len(batch) == 10
|
||||
assert isinstance(batch[0], dict)
|
||||
assert isinstance(batch[0]["a"], torch.Tensor)
|
||||
rows = torch_perm.__getitems__([0, 1, 2])
|
||||
assert isinstance(rows, list)
|
||||
assert len(rows) == 3
|
||||
assert isinstance(rows[0], dict)
|
||||
assert isinstance(rows[0]["a"], torch.Tensor)
|
||||
dataloader = torch.utils.data.DataLoader(torch_perm, batch_size=10, shuffle=True)
|
||||
for batch in dataloader:
|
||||
assert isinstance(batch, dict)
|
||||
assert batch["a"].shape == (10,)
|
||||
# Spawn-based workers exercise the pickle round-trip path: the new
|
||||
# transform-as-list shape must survive pickling so workers produce the
|
||||
# same per-row dicts the parent does.
|
||||
spawn_loader = torch.utils.data.DataLoader(
|
||||
torch_perm,
|
||||
batch_size=10,
|
||||
num_workers=2,
|
||||
multiprocessing_context="spawn",
|
||||
)
|
||||
for batch in spawn_loader:
|
||||
assert isinstance(batch, dict)
|
||||
assert batch["a"].shape == (10,)
|
||||
|
||||
# "torch_row" returns a list of row tensors. Works with the default
|
||||
# DataLoader collate (stacks rows into 2D).
|
||||
row_perm = permutation.with_format("torch_row")
|
||||
dataloader = torch.utils.data.DataLoader(row_perm, batch_size=10, shuffle=True)
|
||||
permutation = permutation.with_format("torch")
|
||||
dataloader = torch.utils.data.DataLoader(permutation, batch_size=10, shuffle=True)
|
||||
for batch in dataloader:
|
||||
assert batch.size(0) == 10
|
||||
assert batch.size(1) == 1
|
||||
|
||||
col_perm = permutation.with_format("torch_col")
|
||||
permutation = permutation.with_format("torch_col")
|
||||
dataloader = torch.utils.data.DataLoader(
|
||||
col_perm, collate_fn=lambda x: x, batch_size=10, shuffle=True
|
||||
permutation, collate_fn=lambda x: x, batch_size=10, shuffle=True
|
||||
)
|
||||
for batch in dataloader:
|
||||
assert batch.size(0) == 1
|
||||
|
||||
@@ -25,42 +25,10 @@ import pandas as pd
|
||||
import polars as pl
|
||||
import pytest
|
||||
import lancedb
|
||||
from lancedb.util import flatten_columns, get_uri_scheme, join_uri, value_to_sql
|
||||
from lancedb.util import get_uri_scheme, join_uri, value_to_sql
|
||||
from utils import exception_output
|
||||
|
||||
|
||||
def _struct_table() -> pa.Table:
|
||||
return pa.table(
|
||||
{
|
||||
"id": [1, 2],
|
||||
"nested": pa.array([{"a": 1, "b": 2}, {"a": 3, "b": 4}]),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_flatten_columns():
|
||||
tbl = _struct_table()
|
||||
|
||||
# None / False mean "do not flatten": the struct column is preserved.
|
||||
# `False` is a regression guard: because bool is a subclass of int it used
|
||||
# to fall into the integer branch and raise ValueError (see issue).
|
||||
for no_flatten in (None, False):
|
||||
result = flatten_columns(tbl, no_flatten)
|
||||
assert result.column_names == ["id", "nested"]
|
||||
|
||||
# True flattens all nested levels.
|
||||
flattened = flatten_columns(tbl, True)
|
||||
assert flattened.column_names == ["id", "nested.a", "nested.b"]
|
||||
|
||||
# A positive integer flattens up to that depth.
|
||||
flattened = flatten_columns(tbl, 1)
|
||||
assert flattened.column_names == ["id", "nested.a", "nested.b"]
|
||||
|
||||
# Non-positive integers are still rejected.
|
||||
with pytest.raises(ValueError):
|
||||
flatten_columns(tbl, 0)
|
||||
|
||||
|
||||
def test_normalize_uri():
|
||||
uris = [
|
||||
"relative/path",
|
||||
|
||||
@@ -27,7 +27,6 @@ pub mod header;
|
||||
pub mod index;
|
||||
pub mod namespace;
|
||||
pub mod oauth;
|
||||
pub mod otel;
|
||||
pub mod permutation;
|
||||
pub mod query;
|
||||
pub mod runtime;
|
||||
@@ -62,15 +61,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<PyAsyncPermutationBuilder>()?;
|
||||
m.add_class::<PyPermutationReader>()?;
|
||||
m.add_class::<PyExpr>()?;
|
||||
// OpenTelemetry metrics bridge
|
||||
m.add_class::<otel::PyMetricPoint>()?;
|
||||
m.add_class::<otel::PyMetricDescription>()?;
|
||||
m.add_function(wrap_pyfunction!(
|
||||
otel::register_lancedb_metrics_recorder,
|
||||
m
|
||||
)?)?;
|
||||
m.add_function(wrap_pyfunction!(otel::lancedb_metrics_catalog, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(otel::snapshot_lancedb_metrics, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(connect, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(connect_namespace, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(connect_namespace_client, m)?)?;
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Python-facing wrappers over [`lancedb::metrics_otel`].
|
||||
//!
|
||||
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
|
||||
//! core crate; this module only converts the core snapshot types into PyO3
|
||||
//! classes and exposes the three entry points to Python, where
|
||||
//! `lancedb/otel.py` bridges them into the user's OpenTelemetry `MeterProvider`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use lancedb::metrics_otel::{MetricPoint, MetricValue};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
/// One metric data point exposed to Python. For counters and gauges only
|
||||
/// `value` is set; for histograms `buckets` (cumulative `le` counts), `count`,
|
||||
/// and `sum` are set.
|
||||
#[pyclass(name = "MetricPoint", get_all)]
|
||||
pub struct PyMetricPoint {
|
||||
name: String,
|
||||
kind: String,
|
||||
attributes: HashMap<String, String>,
|
||||
value: Option<f64>,
|
||||
buckets: Option<Vec<(String, u64)>>,
|
||||
count: Option<u64>,
|
||||
sum: Option<f64>,
|
||||
}
|
||||
|
||||
impl From<MetricPoint> for PyMetricPoint {
|
||||
fn from(point: MetricPoint) -> Self {
|
||||
let kind = point.kind.as_str().to_string();
|
||||
let (value, buckets, count, sum) = match point.value {
|
||||
MetricValue::Scalar(v) => (Some(v), None, None, None),
|
||||
MetricValue::Histogram {
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
} => (None, Some(buckets), Some(count), Some(sum)),
|
||||
};
|
||||
Self {
|
||||
name: point.name,
|
||||
kind,
|
||||
attributes: point.attributes,
|
||||
value,
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A described metric, used by the Python layer to create instruments up front.
|
||||
#[pyclass(name = "MetricDescription", get_all)]
|
||||
pub struct PyMetricDescription {
|
||||
name: String,
|
||||
kind: String,
|
||||
unit: Option<String>,
|
||||
description: String,
|
||||
}
|
||||
|
||||
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
|
||||
///
|
||||
/// Returns `True` if the recorder is installed (now or previously). Returns
|
||||
/// `False` if a *different* recorder is already installed — `metrics` allows
|
||||
/// only one global recorder per process, so LanceDB cannot coexist with another.
|
||||
#[pyfunction]
|
||||
pub fn register_lancedb_metrics_recorder() -> bool {
|
||||
lancedb::metrics_otel::register_metrics_recorder()
|
||||
}
|
||||
|
||||
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
|
||||
#[pyfunction]
|
||||
pub fn lancedb_metrics_catalog() -> Vec<PyMetricDescription> {
|
||||
lancedb::metrics_otel::metrics_catalog()
|
||||
.into_iter()
|
||||
.map(|desc| PyMetricDescription {
|
||||
name: desc.name,
|
||||
kind: desc.kind.as_str().to_string(),
|
||||
unit: desc.unit,
|
||||
description: desc.description,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
|
||||
/// is installed.
|
||||
///
|
||||
/// The read is lock-free but not O(1): it walks every registered series and
|
||||
/// allocates owned copies of their names and labels. The GIL is released across
|
||||
/// that work so a periodic collection doesn't stall other Python threads.
|
||||
#[pyfunction]
|
||||
pub fn snapshot_lancedb_metrics(py: Python<'_>) -> Vec<PyMetricPoint> {
|
||||
let points = py.detach(lancedb::metrics_otel::snapshot_metrics);
|
||||
points.into_iter().map(PyMetricPoint::from).collect()
|
||||
}
|
||||
@@ -322,12 +322,6 @@ impl From<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
|
||||
fn from(inner: lancedb::table::LsmWriteSpec) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
#[pyclass(get_all, from_py_object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AddColumnsResult {
|
||||
@@ -1035,14 +1029,6 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn get_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let spec = inner.get_lsm_write_spec().await.infer_error()?;
|
||||
Ok(spec.map(LsmWriteSpec::from))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import lancedb
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
# The metrics recorder is process-global and installed once, so the whole
|
||||
# bridge is exercised in a single test to avoid cross-test global-state coupling.
|
||||
|
||||
|
||||
def _metrics_by_name(reader):
|
||||
data = reader.get_metrics_data()
|
||||
result = {}
|
||||
for resource_metrics in data.resource_metrics:
|
||||
for scope_metrics in resource_metrics.scope_metrics:
|
||||
for metric in scope_metrics.metrics:
|
||||
result[metric.name] = metric
|
||||
return result
|
||||
|
||||
|
||||
def test_instrument_lancedb_metrics_exports_object_store_metrics(tmp_path):
|
||||
pytest.importorskip("opentelemetry.sdk.metrics")
|
||||
from lancedb.otel import instrument_lancedb_metrics
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
|
||||
reader = InMemoryMetricReader()
|
||||
provider = MeterProvider(metric_readers=[reader])
|
||||
assert instrument_lancedb_metrics(provider)
|
||||
|
||||
# The catalog is populated once the recorder is installed.
|
||||
from lancedb._lancedb import lancedb_metrics_catalog
|
||||
|
||||
catalog = {desc.name: desc for desc in lancedb_metrics_catalog()}
|
||||
# Every metric kind emitted by the object store must be described so it is
|
||||
# surfaced by the bridge (counter, histogram, and gauge).
|
||||
assert catalog["lance_object_store_requests_total"].kind == "counter"
|
||||
assert catalog["lance_object_store_request_duration_seconds"].kind == "histogram"
|
||||
assert catalog["lance_object_store_in_flight_requests"].kind == "gauge"
|
||||
assert catalog["lance_object_store_retryable_responses_total"].kind == "counter"
|
||||
|
||||
# Generate object store activity on the local filesystem (scheme "file").
|
||||
db = lancedb.connect(str(tmp_path))
|
||||
table = db.create_table("t", pa.table({"id": pa.array(range(256))}))
|
||||
assert table.count_rows() == 256
|
||||
assert table.to_arrow().num_rows == 256
|
||||
|
||||
metrics = _metrics_by_name(reader)
|
||||
|
||||
requests = metrics["lance_object_store_requests_total"]
|
||||
points = list(requests.data.data_points)
|
||||
assert points, "expected at least one request data point"
|
||||
# Object store metrics are labelled by `operation` and `base` (the store
|
||||
# scheme, e.g. "file", by default).
|
||||
assert all("base" in p.attributes and "operation" in p.attributes for p in points)
|
||||
assert sum(p.value for p in points) > 0
|
||||
|
||||
# Histograms are decomposed into bucket / count / sum observable counters.
|
||||
bucket = metrics["lance_object_store_request_duration_seconds_bucket"]
|
||||
bucket_points = list(bucket.data.data_points)
|
||||
assert bucket_points
|
||||
assert all("le" in p.attributes for p in bucket_points)
|
||||
# The implicit +Inf bucket must be present and is the cumulative maximum.
|
||||
assert any(p.attributes["le"] == "+Inf" for p in bucket_points)
|
||||
|
||||
count = metrics["lance_object_store_request_duration_seconds_count"]
|
||||
assert sum(p.value for p in count.data.data_points) > 0
|
||||
|
||||
# The `_sum` instrument must also be wired and report positive latency.
|
||||
duration_sum = metrics["lance_object_store_request_duration_seconds_sum"]
|
||||
assert sum(p.value for p in duration_sum.data.data_points) > 0
|
||||
|
||||
# Unit handling: only `_sum` keeps the histogram's unit (seconds); `_bucket`
|
||||
# and `_count` observe cumulative counts and are unitless.
|
||||
assert duration_sum.unit == "s"
|
||||
assert bucket.unit == ""
|
||||
assert count.unit == ""
|
||||
|
||||
|
||||
def test_snapshot_empty_before_install_is_safe():
|
||||
# snapshot is callable regardless of installation state and never raises.
|
||||
from lancedb._lancedb import snapshot_lancedb_metrics
|
||||
|
||||
assert isinstance(snapshot_lancedb_metrics(), list)
|
||||
|
||||
|
||||
def test_instrument_warns_when_recorder_unavailable(monkeypatch):
|
||||
# A foreign `metrics` recorder already installed -> register returns False;
|
||||
# instrument_lancedb_metrics must warn and return False without instrumenting.
|
||||
pytest.importorskip("opentelemetry.sdk.metrics")
|
||||
import lancedb.otel as otel
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
||||
|
||||
monkeypatch.setattr(otel, "register_lancedb_metrics_recorder", lambda: False)
|
||||
|
||||
reader = InMemoryMetricReader()
|
||||
provider = MeterProvider(metric_readers=[reader])
|
||||
with pytest.warns(UserWarning, match="recorder"):
|
||||
assert otel.instrument_lancedb_metrics(provider) is False
|
||||
Generated
+18
-63
@@ -780,7 +780,7 @@ name = "cuda-bindings"
|
||||
version = "13.3.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "cuda-pathfinder", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "cuda-pathfinder" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" },
|
||||
@@ -815,37 +815,37 @@ wheels = [
|
||||
|
||||
[package.optional-dependencies]
|
||||
cublas = [
|
||||
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
cudart = [
|
||||
{ name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
cufft = [
|
||||
{ name = "nvidia-cufft", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
cufile = [
|
||||
{ name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
|
||||
]
|
||||
cupti = [
|
||||
{ name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
curand = [
|
||||
{ name = "nvidia-curand", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
cusolver = [
|
||||
{ name = "nvidia-cusolver", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
cusparse = [
|
||||
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
nvjitlink = [
|
||||
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
nvrtc = [
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
nvtx = [
|
||||
{ name = "nvidia-nvtx", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
|
||||
{ name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1909,9 +1909,6 @@ embeddings = [
|
||||
{ name = "sentencepiece" },
|
||||
{ name = "torch" },
|
||||
]
|
||||
otel = [
|
||||
{ name = "opentelemetry-api" },
|
||||
]
|
||||
pylance = [
|
||||
{ name = "pylance" },
|
||||
]
|
||||
@@ -1926,7 +1923,6 @@ tests = [
|
||||
{ name = "boto3" },
|
||||
{ name = "datafusion" },
|
||||
{ name = "duckdb" },
|
||||
{ name = "opentelemetry-sdk" },
|
||||
{ name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
|
||||
{ name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
|
||||
@@ -1967,8 +1963,6 @@ requires-dist = [
|
||||
{ name = "open-clip-torch", marker = "extra == 'clip'" },
|
||||
{ name = "open-clip-torch", marker = "extra == 'embeddings'", specifier = ">=2.20.0" },
|
||||
{ name = "openai", marker = "extra == 'embeddings'", specifier = ">=1.6.1" },
|
||||
{ name = "opentelemetry-api", marker = "extra == 'otel'" },
|
||||
{ name = "opentelemetry-sdk", marker = "extra == 'tests'", specifier = ">=1.30.0" },
|
||||
{ name = "overrides", marker = "python_full_version < '3.12'", specifier = ">=0.7" },
|
||||
{ name = "packaging", specifier = ">=23.0" },
|
||||
{ name = "pandas", marker = "extra == 'tests'", specifier = ">=1.4" },
|
||||
@@ -2000,7 +1994,7 @@ requires-dist = [
|
||||
{ name = "transformers", marker = "extra == 'siglip'", specifier = ">=4.41.0" },
|
||||
{ name = "typing-extensions", marker = "python_full_version < '3.11' and extra == 'dev'", specifier = ">=4.0.0" },
|
||||
]
|
||||
provides-extras = ["azure", "clip", "dev", "docs", "embeddings", "otel", "pylance", "siglip", "tests"]
|
||||
provides-extras = ["azure", "clip", "dev", "docs", "embeddings", "pylance", "siglip", "tests"]
|
||||
|
||||
[[package]]
|
||||
name = "lomond"
|
||||
@@ -2781,7 +2775,7 @@ name = "nvidia-cudnn-cu13"
|
||||
version = "9.19.0.56"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-cublas" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" },
|
||||
@@ -2793,7 +2787,7 @@ name = "nvidia-cufft"
|
||||
version = "12.0.0.61"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
|
||||
@@ -2823,9 +2817,9 @@ name = "nvidia-cusolver"
|
||||
version = "12.0.4.66"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-cublas" },
|
||||
{ name = "nvidia-cusparse" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
|
||||
@@ -2837,7 +2831,7 @@ name = "nvidia-cusparse"
|
||||
version = "12.6.3.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
|
||||
{ name = "nvidia-nvjitlink" },
|
||||
]
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
|
||||
@@ -2940,45 +2934,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f6/46/180e14be801a75bc13f234cb1b594b232adeb9c84e60a9ab1832e8333591/openai-2.40.0-py3-none-any.whl", hash = "sha256:2b205637ff214477f9ce9ab035e9f494db0e3fa8f1e599008953735fbf6ff1ff", size = 1350935, upload-time = "2026-06-01T21:48:21.462Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-api"
|
||||
version = "1.43.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ae/cc/e4c9584181f86494df0f6bdec1a4f3280c50db44704dc2a407e994fc87bb/opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1", size = 73476, upload-time = "2026-06-24T15:19:55.323Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-sdk"
|
||||
version = "1.43.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "opentelemetry-semantic-conventions" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3e/eb/5041074274ac0956b03637cc039d434569112468e875eddfcc9a0674ce06/opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9", size = 254744, upload-time = "2026-06-24T15:20:08.467Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/49/e3/b17be23af124201c9f52eececd4cc8ddfed1597d37b4ee771895d325805c/opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823", size = 178852, upload-time = "2026-06-24T15:19:52.169Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-semantic-conventions"
|
||||
version = "0.64b0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "opentelemetry-api" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5a/30/5f26df29509eccd86b99b481ac9ffa39da49ba9577cc69071c552ae30447/opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c", size = 148340, upload-time = "2026-06-24T15:20:09.267Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "overrides"
|
||||
version = "7.7.0"
|
||||
|
||||
+1
-18
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.32.0-beta.0"
|
||||
version = "0.31.0-beta.6"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
@@ -49,8 +49,6 @@ lance-encoding = { workspace = true }
|
||||
lance-arrow = { workspace = true }
|
||||
lance-namespace = { workspace = true }
|
||||
lance-namespace-impls = { workspace = true }
|
||||
metrics = { workspace = true, optional = true }
|
||||
metrics-util = { workspace = true, optional = true }
|
||||
moka = { workspace = true }
|
||||
pin-project = { workspace = true }
|
||||
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
|
||||
@@ -130,12 +128,6 @@ azure = [
|
||||
"lance-namespace-impls/dir-azure",
|
||||
"lance-namespace-impls/credential-vendor-azure",
|
||||
]
|
||||
cos = ["lance/tencent", "lance-io/tencent"]
|
||||
goosefs = [
|
||||
"lance/goosefs",
|
||||
"lance-io/goosefs",
|
||||
"lance-namespace-impls/dir-goosefs",
|
||||
]
|
||||
huggingface = [
|
||||
"lance/huggingface",
|
||||
"lance-io/huggingface",
|
||||
@@ -149,15 +141,6 @@ remote = [
|
||||
"lance-namespace-impls/rest",
|
||||
"lance-namespace-impls/rest-adapter",
|
||||
]
|
||||
# Publish LanceDB's internal metrics (currently object store request counts,
|
||||
# bytes, latency, errors, and throttles) through the `metrics` crate facade,
|
||||
# and re-export the `metrics` crate as `lancedb::metrics`. Install any
|
||||
# `metrics`-compatible recorder to collect them.
|
||||
metrics = ["dep:metrics", "lance/metrics", "lance-io/metrics"]
|
||||
# Additional adapter on top of `metrics` that installs a process-global recorder
|
||||
# and exposes a pull-based snapshot/catalog API (see `lancedb::metrics_otel`)
|
||||
# for bridging metrics into OpenTelemetry or other pull-based exporters.
|
||||
metrics-otel = ["metrics", "dep:metrics-util"]
|
||||
fp16kernels = ["lance-linalg/fp16kernels"]
|
||||
s3-test = []
|
||||
bedrock = ["dep:aws-sdk-bedrockruntime"]
|
||||
|
||||
@@ -342,19 +342,6 @@ impl ListingDatabase {
|
||||
))
|
||||
}
|
||||
|
||||
fn storage_base_uri(uri: &str) -> String {
|
||||
let Ok(mut url) = url::Url::parse(uri) else {
|
||||
return uri.to_string();
|
||||
};
|
||||
url.set_query(None);
|
||||
let Some((storage_scheme, _commit_scheme)) = url.scheme().split_once('+') else {
|
||||
return url.to_string();
|
||||
};
|
||||
let storage_scheme = storage_scheme.to_string();
|
||||
let _ = url.set_scheme(&storage_scheme);
|
||||
url.to_string()
|
||||
}
|
||||
|
||||
async fn prepare_namespace_root(
|
||||
uri: &str,
|
||||
storage_options: &HashMap<String, String>,
|
||||
@@ -533,8 +520,6 @@ impl ListingDatabase {
|
||||
// will add a trailing '?' to the url
|
||||
url.set_query(None);
|
||||
|
||||
let storage_base_uri = Self::storage_base_uri(url.as_str());
|
||||
|
||||
let table_base_uri = if let Some(store) = engine {
|
||||
static WARN_ONCE: std::sync::Once = std::sync::Once::new();
|
||||
WARN_ONCE.call_once(|| {
|
||||
@@ -547,6 +532,8 @@ impl ListingDatabase {
|
||||
url.to_string()
|
||||
};
|
||||
|
||||
let plain_uri = url.to_string();
|
||||
|
||||
let session = request
|
||||
.session
|
||||
.clone()
|
||||
@@ -563,13 +550,13 @@ impl ListingDatabase {
|
||||
};
|
||||
let (object_store, base_path) = ObjectStore::from_uri_and_params(
|
||||
session.store_registry(),
|
||||
&storage_base_uri,
|
||||
&plain_uri,
|
||||
&os_params,
|
||||
)
|
||||
.await?;
|
||||
if object_store.is_local() {
|
||||
Self::try_create_dir(&storage_base_uri).context(CreateDirSnafu {
|
||||
path: storage_base_uri.clone(),
|
||||
Self::try_create_dir(&plain_uri).context(CreateDirSnafu {
|
||||
path: plain_uri.clone(),
|
||||
})?;
|
||||
}
|
||||
|
||||
@@ -583,7 +570,7 @@ impl ListingDatabase {
|
||||
};
|
||||
|
||||
let namespace_database = Self::connect_namespace_database(
|
||||
&storage_base_uri,
|
||||
&table_base_uri,
|
||||
options.storage_options.clone(),
|
||||
request.namespace_client_properties.clone(),
|
||||
request.read_consistency_interval,
|
||||
@@ -1322,60 +1309,6 @@ mod tests {
|
||||
(tempdir, db)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_listing_database_root_ops_do_not_create_manifest() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options: Default::default(),
|
||||
namespace_client_properties: Default::default(),
|
||||
manifest_enabled: false,
|
||||
read_consistency_interval: None,
|
||||
session: None,
|
||||
};
|
||||
|
||||
let db = ListingDatabase::connect_with_options(&request)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!tempdir.path().join("__manifest").exists());
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "root_table".to_string(),
|
||||
namespace_path: vec![],
|
||||
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
db.open_table(OpenTableRequest {
|
||||
name: "root_table".to_string(),
|
||||
namespace_path: vec![],
|
||||
index_cache_size: None,
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
#[allow(deprecated)]
|
||||
let table_names = db.table_names(TableNamesRequest::default()).await.unwrap();
|
||||
|
||||
assert_eq!(table_names, vec!["root_table".to_string()]);
|
||||
assert!(!tempdir.path().join("__manifest").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clone_table_basic() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
@@ -2350,24 +2283,6 @@ mod tests {
|
||||
assert_eq!(captured.as_deref(), Some("foo=bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_base_uri_strips_commit_engine_scheme() {
|
||||
assert_eq!(
|
||||
ListingDatabase::storage_base_uri("s3+ddb://bucket/prefix?ddbTableName=commit_table"),
|
||||
"s3://bucket/prefix"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
ListingDatabase::storage_base_uri("s3://bucket/prefix?foo=bar"),
|
||||
"s3://bucket/prefix"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
ListingDatabase::storage_base_uri("/tmp/lancedb"),
|
||||
"/tmp/lancedb"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression: connecting via a URL-style URI (which goes through
|
||||
/// `url::Url::parse` and the `query_pairs_mut()` path) must not
|
||||
/// append a trailing `?` to per-table URIs when the input URI has
|
||||
|
||||
@@ -33,11 +33,6 @@
|
||||
//! - `remote` - Enable remote client to connect to LanceDB cloud.
|
||||
//! - `huggingface` - Enable HuggingFace Hub integration for loading datasets from the Hub.
|
||||
//! - `fp16kernels` - Enable FP16 kernels for faster vector search on CPU.
|
||||
//! - `metrics` - Publish LanceDB's internal metrics through the
|
||||
//! [`metrics`](https://docs.rs/metrics) crate facade and re-export that crate.
|
||||
//! Install any `metrics`-compatible recorder to collect them.
|
||||
//! - `metrics-otel` - Add a pull-based adapter (the `metrics_otel` module) over
|
||||
//! the `metrics` facade for bridging metrics into OpenTelemetry or similar.
|
||||
//!
|
||||
//! ### Quick Start
|
||||
//!
|
||||
@@ -179,8 +174,6 @@ pub mod expr;
|
||||
pub mod index;
|
||||
pub mod io;
|
||||
pub mod ipc;
|
||||
#[cfg(feature = "metrics-otel")]
|
||||
pub mod metrics_otel;
|
||||
#[cfg(feature = "polars")]
|
||||
mod polars_arrow_convertors;
|
||||
pub mod query;
|
||||
@@ -201,12 +194,6 @@ pub use connection::{ConnectNamespaceBuilder, Connection};
|
||||
pub use error::{Error, Result};
|
||||
use lance_index::vector::ApproxMode as LanceApproxMode;
|
||||
use lance_linalg::distance::DistanceType as LanceDistanceType;
|
||||
/// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
|
||||
/// the `metrics` feature to publish LanceDB's internal metrics; install any
|
||||
/// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
|
||||
/// a built-in pull-based adapter.
|
||||
#[cfg(feature = "metrics")]
|
||||
pub use metrics;
|
||||
pub use table::Table;
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, Default)]
|
||||
|
||||
@@ -1,594 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! A pull-based adapter over the [`metrics`] crate facade.
|
||||
//!
|
||||
//! LanceDB (through Lance core) publishes metrics — currently object store
|
||||
//! request counts, bytes, latency, errors, and throttles — through the global
|
||||
//! [`metrics`] facade without choosing a backend. This module installs a
|
||||
//! process-global [`metrics::Recorder`] that aggregates those metrics into
|
||||
//! lock-free cumulative storage and exposes that state as a snapshot, so callers
|
||||
//! can feed it into a pull-based exporter such as OpenTelemetry.
|
||||
//!
|
||||
//! The language bindings build their OpenTelemetry integrations on top of the
|
||||
//! three public entry points here: [`register_metrics_recorder`],
|
||||
//! [`metrics_catalog`], and [`snapshot_metrics`].
|
||||
//!
|
||||
//! The recorder is *generic*: it records any metric emitted through the facade,
|
||||
//! keyed by name and labels. Object store metrics are the first producer, but
|
||||
//! nothing here is specific to them. New metrics flow through automatically;
|
||||
//! they only need to be described (via the `metrics` `describe_*!` macros) so
|
||||
//! callers can discover their name, kind, and unit up front.
|
||||
//!
|
||||
//! ## Why pull, not push
|
||||
//!
|
||||
//! OpenTelemetry collects on its own schedule and invokes observable-instrument
|
||||
//! callbacks at collection time. Cumulative counters map directly onto OTel's
|
||||
//! `ObservableCounter` semantics. So the adapter aggregates in Rust and lets the
|
||||
//! collection thread pull a [`snapshot`](snapshot_metrics) on demand.
|
||||
//!
|
||||
//! ## Histograms
|
||||
//!
|
||||
//! OpenTelemetry has no asynchronous histogram instrument, so histograms cannot
|
||||
//! be pulled as-is. Instead each histogram is aggregated into fixed buckets
|
||||
//! (Prometheus style) and exposed as cumulative `le` bucket counts plus a count
|
||||
//! and sum, which the caller can surface as observable counters.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
|
||||
|
||||
use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
|
||||
use metrics_util::registry::{Registry, Storage};
|
||||
|
||||
/// Bucket boundaries used when a histogram has no registered bounds. Covers a
|
||||
/// broad latency range so unknown histograms still produce useful buckets.
|
||||
const DEFAULT_BOUNDS: &[f64] = &[
|
||||
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
|
||||
];
|
||||
|
||||
/// The kind of a metric, mirroring the three `metrics` instrument types.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MetricKind {
|
||||
Counter,
|
||||
Gauge,
|
||||
Histogram,
|
||||
}
|
||||
|
||||
impl MetricKind {
|
||||
/// The lowercase name of this kind (`"counter"`, `"gauge"`, `"histogram"`).
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Counter => "counter",
|
||||
Self::Gauge => "gauge",
|
||||
Self::Histogram => "histogram",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A described metric, used to create one exporter instrument per metric up front.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MetricDescription {
|
||||
/// The metric name (e.g. `lance_object_store_requests_total`).
|
||||
pub name: String,
|
||||
/// Whether the metric is a counter, gauge, or histogram.
|
||||
pub kind: MetricKind,
|
||||
/// The canonical unit label, if the producer described one.
|
||||
pub unit: Option<String>,
|
||||
/// Human-readable help text describing the metric.
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
/// The aggregated value of a metric at snapshot time.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum MetricValue {
|
||||
/// A counter or gauge value.
|
||||
Scalar(f64),
|
||||
/// A histogram, decomposed into cumulative `le` buckets plus count and sum.
|
||||
Histogram {
|
||||
/// Cumulative `(le, count)` buckets, ending in the implicit `+Inf` bucket.
|
||||
buckets: Vec<(String, u64)>,
|
||||
/// Total number of recorded samples.
|
||||
count: u64,
|
||||
/// Sum of all recorded sample values.
|
||||
sum: f64,
|
||||
},
|
||||
}
|
||||
|
||||
/// One aggregated metric data point exposed to a caller.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MetricPoint {
|
||||
/// The metric name.
|
||||
pub name: String,
|
||||
/// Whether the point is a counter, gauge, or histogram.
|
||||
pub kind: MetricKind,
|
||||
/// The label set for this point (e.g. `operation`, `base`).
|
||||
pub attributes: HashMap<String, String>,
|
||||
/// The aggregated value.
|
||||
pub value: MetricValue,
|
||||
}
|
||||
|
||||
/// Catalog of described metrics, keyed by metric name.
|
||||
static CATALOG: LazyLock<Mutex<HashMap<String, CatalogEntry>>> =
|
||||
LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
struct CatalogEntry {
|
||||
kind: MetricKind,
|
||||
unit: Option<String>,
|
||||
description: String,
|
||||
}
|
||||
|
||||
/// Per-metric histogram bucket boundaries, keyed by metric name. Producers
|
||||
/// register their recommended bounds before any metric is recorded.
|
||||
static HISTOGRAM_BOUNDS: LazyLock<RwLock<HashMap<String, Arc<[f64]>>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
/// The installed recorder's registry, available once installation succeeds.
|
||||
static REGISTRY: OnceLock<Arc<Registry<Key, LanceStorage>>> = OnceLock::new();
|
||||
|
||||
fn bounds_for(name: &str) -> Arc<[f64]> {
|
||||
HISTOGRAM_BOUNDS
|
||||
.read()
|
||||
.unwrap()
|
||||
.get(name)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS))
|
||||
}
|
||||
|
||||
/// A histogram that buckets samples at record time into fixed boundaries,
|
||||
/// keeping a cumulative count and sum. Bucketing eagerly keeps memory bounded
|
||||
/// (unlike retaining raw samples) and produces Prometheus-style `le` buckets.
|
||||
struct BucketedHistogram {
|
||||
/// Sorted, finite upper bounds. A sample `v` falls in the first bucket whose
|
||||
/// bound is `>= v`; samples above all bounds fall in the implicit `+Inf`
|
||||
/// bucket stored as the final entry of `counts`.
|
||||
bounds: Arc<[f64]>,
|
||||
/// Per-bucket (non-cumulative) counts; length is `bounds.len() + 1`.
|
||||
counts: Box<[AtomicU64]>,
|
||||
count: AtomicU64,
|
||||
/// Running sum of recorded values, stored as `f64` bits (there is no atomic
|
||||
/// f64, so the bit pattern is held in a `u64`; see [`Self::add_to_sum`]).
|
||||
sum_bits: AtomicU64,
|
||||
}
|
||||
|
||||
// All atomics here use `Ordering::Relaxed`: each metric counter is independent,
|
||||
// so no happens-before relationship is needed between them, and a snapshot
|
||||
// reader tolerates slightly stale values. This matches `metrics_util`'s
|
||||
// `AtomicStorage`.
|
||||
|
||||
impl BucketedHistogram {
|
||||
fn new(bounds: Arc<[f64]>) -> Self {
|
||||
let counts = (0..bounds.len() + 1)
|
||||
.map(|_| AtomicU64::new(0))
|
||||
.collect::<Vec<_>>()
|
||||
.into_boxed_slice();
|
||||
Self {
|
||||
bounds,
|
||||
counts,
|
||||
count: AtomicU64::new(0),
|
||||
sum_bits: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn add_to_sum(&self, value: f64) {
|
||||
// No atomic offers an f64 add, so read the current bit pattern, add in
|
||||
// float space, and CAS it back, retrying if another thread won the race.
|
||||
let mut current = self.sum_bits.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let updated = (f64::from_bits(current) + value).to_bits();
|
||||
match self.sum_bits.compare_exchange_weak(
|
||||
current,
|
||||
updated,
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => break,
|
||||
Err(actual) => current = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cumulative `le` buckets, total count, and sum at this instant.
|
||||
fn snapshot(&self) -> MetricValue {
|
||||
let mut cumulative = 0u64;
|
||||
let mut buckets = Vec::with_capacity(self.bounds.len() + 1);
|
||||
for (i, bound) in self.bounds.iter().enumerate() {
|
||||
cumulative += self.counts[i].load(Ordering::Relaxed);
|
||||
buckets.push((format!("{}", bound), cumulative));
|
||||
}
|
||||
cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed);
|
||||
buckets.push(("+Inf".to_string(), cumulative));
|
||||
MetricValue::Histogram {
|
||||
buckets,
|
||||
count: self.count.load(Ordering::Relaxed),
|
||||
sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl metrics::HistogramFn for BucketedHistogram {
|
||||
fn record(&self, value: f64) {
|
||||
let idx = self.bounds.partition_point(|&bound| bound < value);
|
||||
self.counts[idx].fetch_add(1, Ordering::Relaxed);
|
||||
self.count.fetch_add(1, Ordering::Relaxed);
|
||||
self.add_to_sum(value);
|
||||
}
|
||||
}
|
||||
|
||||
/// Storage backing the registry. Counters and gauges are plain atomics (as in
|
||||
/// `metrics_util`'s `AtomicStorage`); histograms use [`BucketedHistogram`].
|
||||
struct LanceStorage;
|
||||
|
||||
impl Storage<Key> for LanceStorage {
|
||||
type Counter = Arc<AtomicU64>;
|
||||
type Gauge = Arc<AtomicU64>;
|
||||
type Histogram = Arc<BucketedHistogram>;
|
||||
|
||||
fn counter(&self, _key: &Key) -> Self::Counter {
|
||||
Arc::new(AtomicU64::new(0))
|
||||
}
|
||||
|
||||
fn gauge(&self, _key: &Key) -> Self::Gauge {
|
||||
// The `metrics` facade writes the f64 bit pattern into this `u64` (the
|
||||
// snapshot decodes it with `f64::from_bits`), matching `AtomicStorage`.
|
||||
// `0` decodes to `0.0`, the correct initial value.
|
||||
Arc::new(AtomicU64::new(0))
|
||||
}
|
||||
|
||||
fn histogram(&self, key: &Key) -> Self::Histogram {
|
||||
Arc::new(BucketedHistogram::new(bounds_for(key.name())))
|
||||
}
|
||||
}
|
||||
|
||||
struct LanceRecorder {
|
||||
registry: Arc<Registry<Key, LanceStorage>>,
|
||||
}
|
||||
|
||||
impl LanceRecorder {
|
||||
fn describe(
|
||||
&self,
|
||||
key: KeyName,
|
||||
kind: MetricKind,
|
||||
unit: Option<Unit>,
|
||||
description: SharedString,
|
||||
) {
|
||||
CATALOG.lock().unwrap().insert(
|
||||
key.as_str().to_string(),
|
||||
CatalogEntry {
|
||||
kind,
|
||||
unit: unit.map(|u| u.as_canonical_label().to_string()),
|
||||
description: description.into_owned(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
impl Recorder for LanceRecorder {
|
||||
fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.describe(key, MetricKind::Counter, unit, description);
|
||||
}
|
||||
|
||||
fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.describe(key, MetricKind::Gauge, unit, description);
|
||||
}
|
||||
|
||||
fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
|
||||
self.describe(key, MetricKind::Histogram, unit, description);
|
||||
}
|
||||
|
||||
fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
|
||||
self.registry
|
||||
.get_or_create_counter(key, |c| Counter::from_arc(c.clone()))
|
||||
}
|
||||
|
||||
fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge {
|
||||
self.registry
|
||||
.get_or_create_gauge(key, |g| Gauge::from_arc(g.clone()))
|
||||
}
|
||||
|
||||
fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram {
|
||||
self.registry
|
||||
.get_or_create_histogram(key, |h| Histogram::from_arc(h.clone()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Register the recommended histogram bounds for every metric-emitting
|
||||
/// subsystem. New subsystems add their `histogram_bounds()` here.
|
||||
fn register_bounds() {
|
||||
let mut bounds = HISTOGRAM_BOUNDS.write().unwrap();
|
||||
for (name, values) in lance_io::object_store::metrics::histogram_bounds() {
|
||||
bounds.insert((*name).to_string(), Arc::from(*values));
|
||||
}
|
||||
}
|
||||
|
||||
/// Describe every metric-emitting subsystem so the catalog is populated. Must
|
||||
/// run after the recorder is installed. New subsystems add their
|
||||
/// `describe_metrics()` here.
|
||||
fn describe_all() {
|
||||
lance_io::object_store::metrics::describe_metrics();
|
||||
}
|
||||
|
||||
fn labels(key: &Key) -> HashMap<String, String> {
|
||||
key.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn collect_points(registry: &Registry<Key, LanceStorage>) -> Vec<MetricPoint> {
|
||||
let mut points = Vec::new();
|
||||
for (key, handle) in registry.get_counter_handles() {
|
||||
points.push(MetricPoint {
|
||||
name: key.name().to_string(),
|
||||
kind: MetricKind::Counter,
|
||||
attributes: labels(&key),
|
||||
// OpenTelemetry observations are float; counts stay well within the
|
||||
// f64-exact integer range (2^53), so this cast is lossless in practice.
|
||||
value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64),
|
||||
});
|
||||
}
|
||||
for (key, handle) in registry.get_gauge_handles() {
|
||||
points.push(MetricPoint {
|
||||
name: key.name().to_string(),
|
||||
kind: MetricKind::Gauge,
|
||||
attributes: labels(&key),
|
||||
value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))),
|
||||
});
|
||||
}
|
||||
for (key, handle) in registry.get_histogram_handles() {
|
||||
points.push(MetricPoint {
|
||||
name: key.name().to_string(),
|
||||
kind: MetricKind::Histogram,
|
||||
attributes: labels(&key),
|
||||
value: handle.snapshot(),
|
||||
});
|
||||
}
|
||||
points
|
||||
}
|
||||
|
||||
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
|
||||
///
|
||||
/// Returns `true` if the recorder is installed (now or previously). Returns
|
||||
/// `false` if a *different* recorder is already installed — `metrics` allows
|
||||
/// only one global recorder per process, so LanceDB cannot coexist with another.
|
||||
pub fn register_metrics_recorder() -> bool {
|
||||
if REGISTRY.get().is_some() {
|
||||
return true;
|
||||
}
|
||||
let registry = Arc::new(Registry::new(LanceStorage));
|
||||
let recorder = LanceRecorder {
|
||||
registry: registry.clone(),
|
||||
};
|
||||
// Register bounds *before* installing the recorder. Bounds don't depend on
|
||||
// the recorder, and once it is installed a concurrent histogram emission
|
||||
// could otherwise create a handle with the fallback bounds and keep them for
|
||||
// the process lifetime.
|
||||
register_bounds();
|
||||
match metrics::set_global_recorder(recorder) {
|
||||
Ok(()) => {
|
||||
let _ = REGISTRY.set(registry);
|
||||
// Describe metrics only after install so the `describe_*!` macros
|
||||
// route through this recorder and populate the catalog.
|
||||
describe_all();
|
||||
true
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
|
||||
pub fn metrics_catalog() -> Vec<MetricDescription> {
|
||||
CATALOG
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|(name, entry)| MetricDescription {
|
||||
name: name.clone(),
|
||||
kind: entry.kind,
|
||||
unit: entry.unit.clone(),
|
||||
description: entry.description.clone(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
|
||||
/// is installed. The read is lock-free.
|
||||
pub fn snapshot_metrics() -> Vec<MetricPoint> {
|
||||
let Some(registry) = REGISTRY.get() else {
|
||||
return Vec::new();
|
||||
};
|
||||
collect_points(registry)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use metrics::HistogramFn;
|
||||
|
||||
fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 {
|
||||
buckets
|
||||
.iter()
|
||||
.find(|(b, _)| b == le)
|
||||
.map(|(_, c)| *c)
|
||||
.unwrap_or_else(|| panic!("no bucket with le={le}"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucketed_histogram_records_cumulative_buckets() {
|
||||
let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
|
||||
hist.record(0.05); // le=0.1
|
||||
hist.record(0.5); // le=1
|
||||
hist.record(0.5); // le=1
|
||||
hist.record(50.0); // +Inf
|
||||
|
||||
let MetricValue::Histogram {
|
||||
buckets,
|
||||
count,
|
||||
sum,
|
||||
} = hist.snapshot()
|
||||
else {
|
||||
panic!("expected histogram");
|
||||
};
|
||||
|
||||
// Buckets are cumulative (Prometheus `le` semantics).
|
||||
assert_eq!(bucket_count(&buckets, "0.1"), 1);
|
||||
assert_eq!(bucket_count(&buckets, "1"), 3);
|
||||
assert_eq!(bucket_count(&buckets, "10"), 3);
|
||||
assert_eq!(bucket_count(&buckets, "+Inf"), 4);
|
||||
assert_eq!(count, 4);
|
||||
assert!((sum - 51.05).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucketed_histogram_boundary_is_inclusive() {
|
||||
let hist = BucketedHistogram::new(Arc::from([1.0f64].as_slice()));
|
||||
hist.record(1.0); // exactly the bound -> le=1, not +Inf
|
||||
let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
|
||||
panic!("expected histogram");
|
||||
};
|
||||
assert_eq!(bucket_count(&buckets, "1"), 1);
|
||||
assert_eq!(bucket_count(&buckets, "+Inf"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucketed_histogram_boundary_is_inclusive_mid_range() {
|
||||
// A value equal to a middle bound lands in that bucket, not the next.
|
||||
let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
|
||||
hist.record(1.0);
|
||||
let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
|
||||
panic!("expected histogram");
|
||||
};
|
||||
assert_eq!(bucket_count(&buckets, "0.1"), 0);
|
||||
assert_eq!(bucket_count(&buckets, "1"), 1);
|
||||
assert_eq!(bucket_count(&buckets, "10"), 1); // cumulative, so still 1
|
||||
assert_eq!(bucket_count(&buckets, "+Inf"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorder_aggregates_counters_with_labels() {
|
||||
let registry = Arc::new(Registry::new(LanceStorage));
|
||||
let recorder = LanceRecorder {
|
||||
registry: registry.clone(),
|
||||
};
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
|
||||
.increment(2);
|
||||
metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
|
||||
.increment(3);
|
||||
// A distinct label set must produce a separate point, not merge.
|
||||
metrics::counter!("test_requests_total", "operation" => "put", "scheme" => "gs")
|
||||
.increment(7);
|
||||
});
|
||||
|
||||
let scalar = |attrs: &[(&str, &str)]| {
|
||||
let points = collect_points(®istry);
|
||||
let point = points
|
||||
.into_iter()
|
||||
.find(|p| {
|
||||
p.name == "test_requests_total"
|
||||
&& attrs
|
||||
.iter()
|
||||
.all(|(k, v)| p.attributes.get(*k).map(String::as_str) == Some(*v))
|
||||
})
|
||||
.expect("counter recorded for label set");
|
||||
assert_eq!(point.kind, MetricKind::Counter);
|
||||
match point.value {
|
||||
MetricValue::Scalar(v) => v,
|
||||
_ => panic!("expected scalar"),
|
||||
}
|
||||
};
|
||||
|
||||
// Same labels aggregate; distinct labels stay separate.
|
||||
assert!((scalar(&[("operation", "get"), ("scheme", "s3")]) - 5.0).abs() < 1e-9);
|
||||
assert!((scalar(&[("operation", "put"), ("scheme", "gs")]) - 7.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorder_records_gauges() {
|
||||
let registry = Arc::new(Registry::new(LanceStorage));
|
||||
let recorder = LanceRecorder {
|
||||
registry: registry.clone(),
|
||||
};
|
||||
// Gauges store the f64 bit pattern in a u64; the snapshot must decode it.
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
metrics::gauge!("test_gauge", "scheme" => "s3").set(3.5);
|
||||
});
|
||||
|
||||
let points = collect_points(®istry);
|
||||
let point = points
|
||||
.iter()
|
||||
.find(|p| p.name == "test_gauge")
|
||||
.expect("gauge recorded");
|
||||
assert_eq!(point.kind, MetricKind::Gauge);
|
||||
assert!(matches!(point.value, MetricValue::Scalar(v) if (v - 3.5).abs() < 1e-9));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorder_falls_back_to_default_bounds() {
|
||||
// A histogram with no registered bounds uses DEFAULT_BOUNDS.
|
||||
let name = "test_unregistered_histogram";
|
||||
assert!(!HISTOGRAM_BOUNDS.read().unwrap().contains_key(name));
|
||||
|
||||
let registry = Arc::new(Registry::new(LanceStorage));
|
||||
let recorder = LanceRecorder {
|
||||
registry: registry.clone(),
|
||||
};
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
metrics::histogram!(name).record(0.02);
|
||||
});
|
||||
|
||||
let points = collect_points(®istry);
|
||||
let point = points.iter().find(|p| p.name == name).expect("recorded");
|
||||
let MetricValue::Histogram { buckets, count, .. } = &point.value else {
|
||||
panic!("expected histogram");
|
||||
};
|
||||
assert_eq!(*count, 1);
|
||||
// DEFAULT_BOUNDS yields one bucket per bound plus the implicit `+Inf`.
|
||||
assert_eq!(buckets.len(), DEFAULT_BOUNDS.len() + 1);
|
||||
// 0.02 falls in the le=0.025 bucket (the third DEFAULT_BOUNDS entry).
|
||||
assert_eq!(bucket_count(buckets, "0.025"), 1);
|
||||
assert_eq!(bucket_count(buckets, "0.01"), 0);
|
||||
assert_eq!(bucket_count(buckets, "+Inf"), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorder_uses_registered_histogram_bounds() {
|
||||
let name = "test_recorder_bounds_seconds";
|
||||
HISTOGRAM_BOUNDS
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(name.to_string(), Arc::from([0.1f64, 1.0].as_slice()));
|
||||
|
||||
let registry = Arc::new(Registry::new(LanceStorage));
|
||||
let recorder = LanceRecorder {
|
||||
registry: registry.clone(),
|
||||
};
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
metrics::histogram!(name).record(0.05);
|
||||
metrics::histogram!(name).record(5.0);
|
||||
});
|
||||
|
||||
let points = collect_points(®istry);
|
||||
let point = points.iter().find(|p| p.name == name).expect("recorded");
|
||||
let MetricValue::Histogram { buckets, count, .. } = &point.value else {
|
||||
panic!("expected histogram");
|
||||
};
|
||||
assert_eq!(*count, 2);
|
||||
assert_eq!(bucket_count(buckets, "0.1"), 1);
|
||||
assert_eq!(bucket_count(buckets, "+Inf"), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn describe_populates_catalog() {
|
||||
let name = "test_describe_catalog_total";
|
||||
let registry = Arc::new(Registry::new(LanceStorage));
|
||||
let recorder = LanceRecorder { registry };
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
metrics::describe_counter!(name, Unit::Count, "a test counter");
|
||||
});
|
||||
|
||||
let catalog = CATALOG.lock().unwrap();
|
||||
let entry = catalog.get(name).expect("described");
|
||||
assert_eq!(entry.kind, MetricKind::Counter);
|
||||
assert_eq!(entry.description, "a test counter");
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,6 @@ use crate::table::AddColumnsResult;
|
||||
use crate::table::AddResult;
|
||||
use crate::table::DeleteResult;
|
||||
use crate::table::DropColumnsResult;
|
||||
use crate::table::LsmWriteSpec;
|
||||
use crate::table::MergeResult;
|
||||
use crate::table::Tags;
|
||||
use crate::table::UpdateResult;
|
||||
@@ -2270,7 +2269,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
})
|
||||
}
|
||||
|
||||
async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> {
|
||||
async fn set_lsm_write_spec(&self, spec: crate::table::LsmWriteSpec) -> Result<()> {
|
||||
use crate::table::LsmWriteSpec;
|
||||
self.check_mutable().await?;
|
||||
|
||||
// Map the spec onto the server's request DTO. `sharding` is internally
|
||||
@@ -2322,69 +2322,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
|
||||
// Read counterpart to set/unset, resolved server-side against HEAD. The
|
||||
// server reads the spec from the `__lance_mem_wal` system index (shard
|
||||
// column mapped from its Lance field id against the current schema) and
|
||||
// re-encodes it into the same sophon-owned shape the set endpoint
|
||||
// accepts — no lance/lancedb types cross the wire. `lsm_write_spec` is
|
||||
// null when the LSM write path is not enabled for the table.
|
||||
let request = self.post_read(&format!(
|
||||
"/v1/table/{}/get_lsm_write_spec/",
|
||||
self.identifier
|
||||
));
|
||||
let (request_id, response) = self.send(request, true).await?;
|
||||
let response = self.check_table_response(&request_id, response).await?;
|
||||
let body = response.text().await.err_to_http(request_id.clone())?;
|
||||
|
||||
// Mirror of sophon's `Sharding` (internally tagged on `mode`) and
|
||||
// `LsmWriteSpecBody` / `GetLsmWriteSpecResponse`.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(tag = "mode", rename_all = "snake_case")]
|
||||
enum Sharding {
|
||||
Unsharded,
|
||||
Bucket { column: String, num_buckets: u32 },
|
||||
Identity { column: String },
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct LsmWriteSpecBody {
|
||||
sharding: Sharding,
|
||||
#[serde(default)]
|
||||
maintained_indexes: Vec<String>,
|
||||
#[serde(default)]
|
||||
writer_config_defaults: std::collections::HashMap<String, String>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct GetLsmWriteSpecResponse {
|
||||
lsm_write_spec: Option<LsmWriteSpecBody>,
|
||||
}
|
||||
|
||||
let parsed: GetLsmWriteSpecResponse =
|
||||
serde_json::from_str(&body).map_err(|e| Error::Http {
|
||||
source: format!("Failed to parse get_lsm_write_spec response: {}", e).into(),
|
||||
request_id,
|
||||
status_code: None,
|
||||
})?;
|
||||
|
||||
let Some(body) = parsed.lsm_write_spec else {
|
||||
// The LSM write path is not enabled for this table.
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let spec = match body.sharding {
|
||||
Sharding::Bucket {
|
||||
column,
|
||||
num_buckets,
|
||||
} => LsmWriteSpec::bucket(column, num_buckets),
|
||||
Sharding::Identity { column } => LsmWriteSpec::identity(column),
|
||||
Sharding::Unsharded => LsmWriteSpec::unsharded(),
|
||||
}
|
||||
.with_maintained_indexes(body.maintained_indexes)
|
||||
.with_writer_config_defaults(body.writer_config_defaults);
|
||||
|
||||
Ok(Some(spec))
|
||||
}
|
||||
|
||||
async fn tags(&self) -> Result<Box<dyn Tags + '_>> {
|
||||
Ok(Box::new(RemoteTags { inner: self }))
|
||||
}
|
||||
@@ -5424,74 +5361,6 @@ mod tests {
|
||||
table.unset_lsm_write_spec().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_lsm_write_spec() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(request.method(), "POST");
|
||||
assert_eq!(
|
||||
request.url().path(),
|
||||
"/v1/table/my_table/get_lsm_write_spec/"
|
||||
);
|
||||
|
||||
// The server resolves the spec and re-encodes it into the same
|
||||
// sophon-owned shape the set endpoint accepts (`Sharding` internally
|
||||
// tagged on `mode`, wrapped in `lsm_write_spec`).
|
||||
let response = serde_json::json!({
|
||||
"lsm_write_spec": {
|
||||
"sharding": { "mode": "bucket", "column": "id", "num_buckets": 4 },
|
||||
"maintained_indexes": ["id_idx"],
|
||||
"writer_config_defaults": { "durable_write": "false" },
|
||||
}
|
||||
});
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(response.to_string())
|
||||
.unwrap()
|
||||
});
|
||||
|
||||
let spec = table
|
||||
.get_lsm_write_spec()
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("a spec should be reported");
|
||||
match spec {
|
||||
crate::table::LsmWriteSpec::Bucket {
|
||||
column,
|
||||
num_buckets,
|
||||
maintained_indexes,
|
||||
writer_config_defaults,
|
||||
} => {
|
||||
assert_eq!(column, "id");
|
||||
assert_eq!(num_buckets, 4);
|
||||
assert_eq!(maintained_indexes, vec!["id_idx".to_string()]);
|
||||
assert_eq!(
|
||||
writer_config_defaults
|
||||
.get("durable_write")
|
||||
.map(String::as_str),
|
||||
Some("false")
|
||||
);
|
||||
}
|
||||
other => panic!("expected a bucket spec, got {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_lsm_write_spec_absent() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(
|
||||
request.url().path(),
|
||||
"/v1/table/my_table/get_lsm_write_spec/"
|
||||
);
|
||||
// Null spec → the LSM write path is not enabled.
|
||||
let response = serde_json::json!({ "lsm_write_spec": null });
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(response.to_string())
|
||||
.unwrap()
|
||||
});
|
||||
assert!(table.get_lsm_write_spec().await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_wait_for_index() {
|
||||
let table = _make_table_with_indices(0);
|
||||
|
||||
@@ -581,16 +581,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
||||
message: "unset_lsm_write_spec is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Read the [`LsmWriteSpec`] currently installed on this table, returning
|
||||
/// `None` when the MemWAL LSM write path is not enabled.
|
||||
///
|
||||
/// The default implementation returns `NotSupported`. Implementations that
|
||||
/// support the MemWAL LSM write path must override this.
|
||||
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
|
||||
Err(Error::NotSupported {
|
||||
message: "get_lsm_write_spec is not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Drain and close any cached MemWAL shard writers for this table.
|
||||
///
|
||||
/// The default implementation is a no-op; table types that maintain
|
||||
@@ -1548,29 +1538,6 @@ impl Table {
|
||||
self.inner.unset_lsm_write_spec().await
|
||||
}
|
||||
|
||||
/// Read the [`LsmWriteSpec`] currently installed on this table.
|
||||
///
|
||||
/// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no
|
||||
/// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]).
|
||||
/// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and
|
||||
/// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to
|
||||
/// [`Table::set_lsm_write_spec`].
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// # use lancedb::table::Table;
|
||||
/// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// if let Some(spec) = table.get_lsm_write_spec().await? {
|
||||
/// println!("LSM write path enabled: {:?}", spec);
|
||||
/// }
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
|
||||
self.inner.get_lsm_write_spec().await
|
||||
}
|
||||
|
||||
/// Drain and close any cached MemWAL shard writers held for this table.
|
||||
///
|
||||
/// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard
|
||||
@@ -2883,10 +2850,6 @@ impl BaseTable for NativeTable {
|
||||
merge::lsm::unset_lsm_write_spec(self).await
|
||||
}
|
||||
|
||||
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
|
||||
merge::lsm::get_lsm_write_spec(self).await
|
||||
}
|
||||
|
||||
async fn close_lsm_writers(&self) -> Result<()> {
|
||||
merge::lsm::close_lsm_writers(self).await
|
||||
}
|
||||
@@ -4448,67 +4411,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_lsm_write_spec() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![
|
||||
Field::new("id", DataType::Int64, false),
|
||||
Field::new("region", DataType::Utf8, true),
|
||||
]));
|
||||
let batch = RecordBatch::try_new(
|
||||
schema.clone(),
|
||||
vec![
|
||||
Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])),
|
||||
Arc::new(StringArray::from(vec!["a", "b", "c"])),
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
|
||||
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
|
||||
let conn = ConnectBuilder::new(uri)
|
||||
.read_consistency_interval(Duration::from_secs(0))
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let table = conn.create_table("t", reader).execute().await.unwrap();
|
||||
|
||||
// No spec installed yet.
|
||||
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
|
||||
|
||||
// A real scalar index is needed to name it as a maintained index.
|
||||
table
|
||||
.create_index(&["id"], Index::Auto)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
let idx_name = table.list_indices().await.unwrap()[0].name.clone();
|
||||
|
||||
// Bucket spec round-trips exactly, including the routing column (recovered
|
||||
// from its field id), maintained indexes, and writer config defaults.
|
||||
let spec = LsmWriteSpec::bucket("id", 4)
|
||||
.with_maintained_indexes([idx_name])
|
||||
.with_writer_config_defaults([("durable_write", "false")]);
|
||||
table.set_lsm_write_spec(spec.clone()).await.unwrap();
|
||||
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
|
||||
|
||||
// After unset, no spec is reported.
|
||||
table.unset_lsm_write_spec().await.unwrap();
|
||||
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
|
||||
|
||||
// Identity sharding round-trips (column recovered from the schema).
|
||||
let spec = LsmWriteSpec::identity("region");
|
||||
table.set_lsm_write_spec(spec.clone()).await.unwrap();
|
||||
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
|
||||
table.unset_lsm_write_spec().await.unwrap();
|
||||
|
||||
// Unsharded round-trips (no routing column).
|
||||
let spec = LsmWriteSpec::unsharded();
|
||||
table.set_lsm_write_spec(spec.clone()).await.unwrap();
|
||||
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
pub async fn test_stats() {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
|
||||
@@ -32,7 +32,7 @@ use lance::dataset::mem_wal::{
|
||||
};
|
||||
use lance::index::DatasetIndexExt;
|
||||
use lance_core::datatypes::Schema as LanceSchema;
|
||||
use lance_index::mem_wal::{MemWalIndexDetails, ShardingField, ShardingSpec};
|
||||
use lance_index::mem_wal::{MemWalIndexDetails, ShardingSpec};
|
||||
use tokio::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -148,97 +148,6 @@ pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// get_lsm_write_spec
|
||||
// =============================================================================
|
||||
|
||||
/// Read the [`LsmWriteSpec`] currently installed on the table, if any.
|
||||
///
|
||||
/// Reconstructs the spec from the MemWAL index's stored details — the same
|
||||
/// metadata [`set_lsm_write_spec`] writes — read directly via
|
||||
/// [`mem_wal_index_details`](DatasetMemWalExt::mem_wal_index_details) (the raw
|
||||
/// manifest record, not the `describe_indices` enrichment path, so it is
|
||||
/// unaffected by system-index filtering there). Returns `Ok(None)` when no
|
||||
/// MemWAL index is installed.
|
||||
#[allow(clippy::redundant_pub_crate)]
|
||||
pub(crate) async fn get_lsm_write_spec(table: &NativeTable) -> Result<Option<LsmWriteSpec>> {
|
||||
let dataset = table.dataset.get().await?;
|
||||
let Some(details) = dataset.mem_wal_index_details().await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
let spec = lsm_write_spec_from_details(&details, dataset.schema())?;
|
||||
Ok(Some(spec))
|
||||
}
|
||||
|
||||
/// Reconstruct the public [`LsmWriteSpec`] from stored MemWAL index details.
|
||||
///
|
||||
/// The sharding transform and its parameters recover the mode; the routing
|
||||
/// column is resolved from the sharding field's source id back to its schema
|
||||
/// name (only bucket / identity carry a column). `maintained_indexes` and
|
||||
/// `writer_config_defaults` are copied verbatim, so the result round-trips
|
||||
/// what `set_lsm_write_spec` installed.
|
||||
fn lsm_write_spec_from_details(
|
||||
details: &MemWalIndexDetails,
|
||||
schema: &LanceSchema,
|
||||
) -> Result<LsmWriteSpec> {
|
||||
let spec = details
|
||||
.sharding_specs
|
||||
.first()
|
||||
.ok_or_else(|| Error::Runtime {
|
||||
message: "get_lsm_write_spec: MemWAL index has no sharding spec".to_string(),
|
||||
})?;
|
||||
let field = spec.fields.first().ok_or_else(|| Error::Runtime {
|
||||
message: "get_lsm_write_spec: MemWAL index has an empty sharding spec".to_string(),
|
||||
})?;
|
||||
|
||||
let base = match field.transform.as_deref() {
|
||||
Some(BUCKET_TRANSFORM) => {
|
||||
let num_buckets = field
|
||||
.parameters
|
||||
.get(NUM_BUCKETS_PARAM)
|
||||
.and_then(|s| s.parse::<u32>().ok())
|
||||
.filter(|n| *n > 0)
|
||||
.ok_or_else(|| Error::Runtime {
|
||||
message: "get_lsm_write_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(),
|
||||
})?;
|
||||
LsmWriteSpec::bucket(sharding_column(field, schema)?, num_buckets)
|
||||
}
|
||||
Some(IDENTITY_TRANSFORM) => LsmWriteSpec::identity(sharding_column(field, schema)?),
|
||||
Some(UNSHARDED_TRANSFORM) => LsmWriteSpec::unsharded(),
|
||||
other => {
|
||||
return Err(Error::Runtime {
|
||||
message: format!(
|
||||
"get_lsm_write_spec: MemWAL index has an unsupported sharding transform {:?}",
|
||||
other
|
||||
),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
Ok(base
|
||||
.with_maintained_indexes(details.maintained_indexes.clone())
|
||||
.with_writer_config_defaults(details.writer_config_defaults.clone()))
|
||||
}
|
||||
|
||||
/// Resolve the single routing column name from a sharding field's source id.
|
||||
///
|
||||
/// `set_lsm_write_spec` records the shard column by its Lance field id, so the
|
||||
/// name is recovered by looking that id up in the current schema.
|
||||
fn sharding_column(field: &ShardingField, schema: &LanceSchema) -> Result<String> {
|
||||
let source_id = *field.source_ids.first().ok_or_else(|| Error::Runtime {
|
||||
message: "get_lsm_write_spec: sharding field has no source column".to_string(),
|
||||
})?;
|
||||
schema
|
||||
.field_by_id(source_id)
|
||||
.map(|f| f.name.clone())
|
||||
.ok_or_else(|| Error::Runtime {
|
||||
message: format!(
|
||||
"get_lsm_write_spec: sharding source field id {} not found in schema",
|
||||
source_id
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// close_lsm_writers
|
||||
// =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user