Compare commits

..

1 Commits

Author SHA1 Message Date
lancedb automation 5e58dde32d chore: update lance dependency to v9.0.0-beta.16 2026-07-06 21:10:10 +00:00
49 changed files with 177 additions and 2793 deletions
-21
View File
@@ -1,21 +0,0 @@
# CODEOWNERS
#
# These owners will be the default owners for everything in the repo.
# They will be requested for review when someone opens a pull request.
#
# See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
# Default owners for everything
* @jackye1995 @wjones127
# Release and publish workflows — changes here can affect supply chain security
/.github/workflows/ @jackye1995 @wjones127 @Xuanwo
# Remote client and auth — sensitive networking and auth code
/rust/lancedb/src/remote/ @jackye1995 @wjones127
# Python FFI boundary
/python/src/ @jackye1995 @wjones127 @AyushExel
# NodeJS FFI boundary
/nodejs/src/ @jackye1995 @wjones127
+9 -11
View File
@@ -34,16 +34,15 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
target: x86_64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
&& unzip /tmp/protoc.zip -d /usr/local \
&& rm /tmp/protoc.zip
- name: Build Arm Manylinux Wheel
if: ${{ inputs.arm-build == 'true' }}
uses: PyO3/maturin-action@v1
@@ -51,14 +50,13 @@ runs:
maturin-version: "1.12.4"
command: build
working-directory: python
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
target: aarch64-unknown-linux-gnu
manylinux: ${{ inputs.manylinux }}
args: ${{ inputs.args }}
before-script-linux: |
set -e
yum install -y clang
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
unzip /tmp/protoc.zip -d /usr/local
rm /tmp/protoc.zip
/usr/local/bin/protoc --version
yum install -y clang \
&& curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
&& unzip /tmp/protoc.zip -d /usr/local \
&& rm /tmp/protoc.zip
Generated
+53 -289
View File
@@ -2039,9 +2039,9 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.20"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
@@ -3151,12 +3151,6 @@ dependencies = [
"encoding_rs",
]
[[package]]
name = "endian-type"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "enum-as-inner"
version = "0.6.1"
@@ -3429,8 +3423,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"rand 0.9.4",
@@ -3785,30 +3779,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "goosefs-sdk"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f"
dependencies = [
"async-trait",
"bytes",
"dashmap",
"hostname",
"prost",
"prost-types",
"rand 0.9.4",
"reqwest 0.12.28",
"serde",
"thiserror 2.0.18",
"tokio",
"tokio-stream",
"tonic",
"tonic-prost",
"tracing",
"uuid",
]
[[package]]
name = "group"
version = "0.12.1"
@@ -4020,17 +3990,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "hostname"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd"
dependencies = [
"cfg-if 1.0.4",
"libc",
"windows-link",
]
[[package]]
name = "http"
version = "0.2.12"
@@ -4191,19 +4150,6 @@ dependencies = [
"webpki-roots 1.0.7",
]
[[package]]
name = "hyper-timeout"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
dependencies = [
"hyper 1.9.0",
"hyper-util",
"pin-project-lite",
"tokio",
"tower-service",
]
[[package]]
name = "hyper-util"
version = "0.1.20"
@@ -4727,7 +4673,7 @@ dependencies = [
"jiff",
"nom 8.0.0",
"num-traits",
"ordered-float 5.3.0",
"ordered-float",
"rand 0.9.4",
"serde",
"serde_json",
@@ -4780,8 +4726,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arc-swap",
"arrow",
@@ -4855,8 +4801,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4865,7 +4811,6 @@ dependencies = [
"arrow-ord",
"arrow-schema",
"arrow-select",
"bytemuck",
"bytes",
"futures",
"getrandom 0.2.17",
@@ -4878,7 +4823,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4892,7 +4837,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4901,8 +4846,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrayref",
"crunchy",
@@ -4912,8 +4857,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4951,8 +4896,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow",
"arrow-array",
@@ -4982,8 +4927,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow",
"arrow-array",
@@ -5000,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"proc-macro2",
"quote",
@@ -5010,8 +4955,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5046,8 +4991,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5077,8 +5022,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arc-swap",
"arrow",
@@ -5143,8 +5088,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow",
"arrow-arith",
@@ -5168,7 +5113,6 @@ dependencies = [
"lance-core",
"lance-namespace",
"log",
"metrics",
"moka",
"object_store",
"object_store_opendal",
@@ -5186,8 +5130,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5203,8 +5147,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow",
"async-trait",
@@ -5216,8 +5160,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5240,7 +5184,7 @@ dependencies = [
"lance-table",
"log",
"object_store",
"quick-xml 0.40.1",
"quick-xml 0.38.4",
"rand 0.9.4",
"reqwest 0.12.28",
"roaring",
@@ -5271,8 +5215,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5287,8 +5231,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow",
"arrow-array",
@@ -5327,8 +5271,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5341,8 +5285,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "9.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
version = "9.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.16#78a814b6b448b433f5ebfdb9a8dbdec7b904c59f"
dependencies = [
"icu_segmenter",
"jieba-rs",
@@ -5411,8 +5355,6 @@ dependencies = [
"lance-testing",
"lazy_static",
"log",
"metrics",
"metrics-util",
"moka",
"num-traits",
"object_store",
@@ -5870,36 +5812,6 @@ dependencies = [
"stable_deref_trait",
]
[[package]]
name = "metrics"
version = "0.24.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2"
dependencies = [
"portable-atomic",
"rapidhash",
]
[[package]]
name = "metrics-util"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376"
dependencies = [
"aho-corasick",
"crossbeam-epoch",
"crossbeam-utils",
"hashbrown 0.15.5",
"indexmap 2.14.0",
"metrics",
"ordered-float 4.6.0",
"quanta",
"radix_trie",
"rand 0.9.4",
"rand_xoshiro",
"sketches-ddsketch",
]
[[package]]
name = "mime"
version = "0.3.17"
@@ -6115,15 +6027,6 @@ dependencies = [
"rawpointer",
]
[[package]]
name = "nibble_vec"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
dependencies = [
"smallvec",
]
[[package]]
name = "nix"
version = "0.26.4"
@@ -6439,9 +6342,7 @@ dependencies = [
"opendal-layer-timeout",
"opendal-service-azblob",
"opendal-service-azdls",
"opendal-service-cos",
"opendal-service-gcs",
"opendal-service-goosefs",
"opendal-service-hf",
"opendal-service-oss",
"opendal-service-s3",
@@ -6569,23 +6470,6 @@ dependencies = [
"opendal-core",
]
[[package]]
name = "opendal-service-cos"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901"
dependencies = [
"bytes",
"http 1.4.2",
"log",
"opendal-core",
"quick-xml 0.39.4",
"reqsign-core",
"reqsign-file-read-tokio",
"reqsign-tencent-cos",
"serde",
]
[[package]]
name = "opendal-service-gcs"
version = "0.57.0"
@@ -6607,20 +6491,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "opendal-service-goosefs"
version = "0.57.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4"
dependencies = [
"bytes",
"goosefs-sdk",
"log",
"opendal-core",
"serde",
"tokio",
]
[[package]]
name = "opendal-service-hf"
version = "0.57.0"
@@ -6688,15 +6558,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordered-float"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951"
dependencies = [
"num-traits",
]
[[package]]
name = "ordered-float"
version = "5.3.0"
@@ -7736,21 +7597,6 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "quanta"
version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
dependencies = [
"crossbeam-utils",
"libc",
"once_cell",
"raw-cpuid",
"wasi 0.11.1+wasi-snapshot-preview1",
"web-sys",
"winapi",
]
[[package]]
name = "quick-xml"
version = "0.26.0"
@@ -7760,6 +7606,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "quick-xml"
version = "0.38.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
dependencies = [
"memchr",
]
[[package]]
name = "quick-xml"
version = "0.39.4"
@@ -7770,15 +7625,6 @@ dependencies = [
"serde",
]
[[package]]
name = "quick-xml"
version = "0.40.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
dependencies = [
"memchr",
]
[[package]]
name = "quinn"
version = "0.11.9"
@@ -7862,16 +7708,6 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]]
name = "radix_trie"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
dependencies = [
"endian-type",
"nibble_vec",
]
[[package]]
name = "rancor"
version = "0.1.1"
@@ -8006,15 +7842,6 @@ version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68"
[[package]]
name = "rapidhash"
version = "4.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e"
dependencies = [
"rustversion",
]
[[package]]
name = "raw-cpuid"
version = "11.6.0"
@@ -8299,21 +8126,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "reqsign-tencent-cos"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e128f19525861dbded59e1e7c17653a8ed63d573ca04aed708d552dbef5bb32a"
dependencies = [
"anyhow",
"http 1.4.2",
"log",
"percent-encoding",
"reqsign-core",
"serde",
"serde_json",
]
[[package]]
name = "reqwest"
version = "0.12.28"
@@ -9192,12 +9004,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
[[package]]
name = "sketches-ddsketch"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
[[package]]
name = "slab"
version = "0.4.12"
@@ -10005,45 +9811,6 @@ dependencies = [
"winnow",
]
[[package]]
name = "tonic"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"h2 0.4.14",
"http 1.4.2",
"http-body 1.0.1",
"http-body-util",
"hyper 1.9.0",
"hyper-timeout",
"hyper-util",
"percent-encoding",
"pin-project",
"socket2 0.6.3",
"sync_wrapper",
"tokio",
"tokio-stream",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tonic-prost"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0"
dependencies = [
"bytes",
"prost",
"tonic",
]
[[package]]
name = "tower"
version = "0.5.3"
@@ -10052,12 +9819,9 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
"indexmap 2.14.0",
"pin-project-lite",
"slab",
"sync_wrapper",
"tokio",
"tokio-util",
"tower-layer",
"tower-service",
"tracing",
+14 -16
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=9.0.0-beta.16", default-features = false, "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.0.0-beta.16", default-features = false, "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.0.0-beta.16", default-features = false, "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.0.0-beta.16", "tag" = "v9.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
@@ -54,8 +54,6 @@ half = { "version" = "2.7.1", default-features = false, features = [
] }
futures = "0"
log = "0.4"
metrics = "0.24"
metrics-util = "0.19"
moka = { version = "0.12", features = ["future"] }
object_store = "0.13.2"
pin-project = "1.0.7"
+30
View File
@@ -51,6 +51,18 @@ ignore = [
# https://rustsec.org/advisories/RUSTSEC-2024-0436
{ id = "RUSTSEC-2024-0436", reason = "transitive via datafusion; awaiting ecosystem migration" },
# encoding: unmaintained. Reached through lindera-dictionary, which is
# required by the native Lindera tokenizer path. Lindera has not migrated
# off this crate yet.
# https://rustsec.org/advisories/RUSTSEC-2021-0153
{ id = "RUSTSEC-2021-0153", reason = "transitive via lindera-dictionary for native Lindera tokenizer" },
# fast-float: unsound and unmaintained. Reached only through polars-arrow
# from the optional Polars integration; replacement requires a Polars
# dependency upgrade.
# https://rustsec.org/advisories/RUSTSEC-2024-0379
{ id = "RUSTSEC-2024-0379", reason = "transitive via polars-arrow; waiting on Polars migration" },
# tantivy: segfault on malformed input due to missing bounds check.
# Pulled in via lance for full-text search. We only feed tantivy
# documents we construct ourselves, not attacker-controlled bytes.
@@ -68,6 +80,18 @@ ignore = [
# https://rustsec.org/advisories/RUSTSEC-2025-0119
{ id = "RUSTSEC-2025-0119", reason = "transitive via hf-hub/indicatif; cosmetic formatting crate" },
# bincode: unmaintained. Reached through lindera and lindera-dictionary,
# which are required by the native Lindera tokenizer path. Lindera has not
# migrated to another serialization format yet.
# https://rustsec.org/advisories/RUSTSEC-2025-0141
{ id = "RUSTSEC-2025-0141", reason = "transitive via lindera/lindera-dictionary for native Lindera tokenizer" },
# lru: soundness issue in IterMut. Reached only through aws-sdk-s3 in
# LanceDB's dev-dependency graph; LanceDB does not use that iterator
# directly. Clearing this requires the AWS SDK chain to update lru.
# https://rustsec.org/advisories/RUSTSEC-2026-0002
{ id = "RUSTSEC-2026-0002", reason = "transitive via aws-sdk-s3 dev-dependency; waiting on AWS SDK lru upgrade" },
# rustls-webpki 0.101.7 (old major line): name-constraint checks for
# URI / wildcard names. Pulled in only via the legacy rustls 0.21 chain
# from aws-smithy-http-client. The 0.103 line we actively use is patched.
@@ -84,6 +108,12 @@ ignore = [
# https://rustsec.org/advisories/RUSTSEC-2026-0104
{ id = "RUSTSEC-2026-0104", reason = "only affects rustls-webpki 0.101 from legacy aws-smithy/rustls 0.21 chain" },
# rand 0.8.5: soundness issue only when ThreadRng reseeds inside a custom
# logger. Reached through several transitive chains. LanceDB does not use
# rand from a custom logger; upgrade once all pinned chains accept 0.8.6+.
# https://rustsec.org/advisories/RUSTSEC-2026-0097
{ id = "RUSTSEC-2026-0097", reason = "transitive rand 0.8.5; LanceDB does not call ThreadRng from custom logging" },
# pyo3 advisories in the Python bindings; tracked pending a patched pyo3 release.
# https://rustsec.org/advisories/RUSTSEC-2026-0176
# https://rustsec.org/advisories/RUSTSEC-2026-0177
-20
View File
@@ -398,26 +398,6 @@ Drop an index from the table.
***
### getLsmWriteSpec()
```ts
abstract getLsmWriteSpec(): Promise<undefined | LsmWriteSpec>
```
Read the [LsmWriteSpec](../interfaces/LsmWriteSpec.md) currently installed on this table.
Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with [Table#unsetLsmWriteSpec](Table.md#unsetlsmwritespec)).
The returned spec — including its `maintainedIndexes` and
`writerConfigDefaults` — mirrors what was passed to
[Table#setLsmWriteSpec](Table.md#setlsmwritespec).
#### Returns
`Promise`&lt;`undefined` \| [`LsmWriteSpec`](../interfaces/LsmWriteSpec.md)&gt;
***
### indexStats()
```ts
@@ -1,42 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / instrumentLanceDbMetrics
# Function: instrumentLanceDbMetrics()
```ts
function instrumentLanceDbMetrics(meterProvider?): boolean
```
Register LanceDB metrics as OpenTelemetry observable instruments.
Installs a process-global metrics recorder and creates one observable
instrument per LanceDB metric (currently object store request counts, bytes,
latency, errors, and throttles) on the given (or global) `MeterProvider`. The
configured `MetricReader` then collects them on its own schedule.
Counters and gauges map directly to observable counters/gauges. Because
OpenTelemetry has no asynchronous histogram instrument, each histogram is
exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
with an `le` attribute) plus `<name>_count` and `<name>_sum`.
Requires `@opentelemetry/api` (a dependency) and, to actually export, an
OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
## Parameters
* **meterProvider?**: `MeterProvider`
The provider to register instruments on. Defaults to the
global provider from `@opentelemetry/api`.
## Returns
`boolean`
`true` if the recorder is installed and instruments are registered.
`false` if a different `metrics` recorder is already installed in this
process (only one global recorder is permitted), in which case a warning is
emitted and no instruments are created. Calling this more than once is safe;
instruments are created only on the first successful call.
-1
View File
@@ -131,7 +131,6 @@
- [RecordBatchIterator](functions/RecordBatchIterator.md)
- [connect](functions/connect.md)
- [connectNamespace](functions/connectNamespace.md)
- [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md)
- [makeArrowTable](functions/makeArrowTable.md)
- [packBits](functions/packBits.md)
- [permutationBuilder](functions/permutationBuilder.md)
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>9.0.0-beta.19</lance-core.version>
<lance-core.version>9.0.0-beta.16</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -44,6 +44,6 @@ aws-lc-rs = "=1.16.3"
napi-build = "2.3.1"
[features]
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/goosefs", "lancedb/metrics-otel"]
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
fp16kernels = ["lancedb/fp16kernels"]
remote = ["lancedb/remote"]
-114
View File
@@ -1,114 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
MeterProvider,
type MetricData,
MetricReader,
} from "@opentelemetry/sdk-metrics";
import * as tmp from "tmp";
import { connect, instrumentLanceDbMetrics } from "../lancedb";
// snapshotLancedbMetrics is internal plumbing (not part of the public API), so
// it is imported from the native module rather than the package entry point.
import { snapshotLancedbMetrics } from "../lancedb/native";
// The metrics recorder is process-global and installed once, so the whole
// bridge is exercised in a single test to avoid cross-test global-state coupling.
// A minimal pull-based reader whose `collect()` we drive directly, invoking the
// observable-instrument callbacks. `@opentelemetry/sdk-metrics` ships no
// in-memory reader, so we subclass the abstract base.
class TestMetricReader extends MetricReader {
protected async onForceFlush(): Promise<void> {
// no-op: collection is driven directly via collect()
}
protected async onShutdown(): Promise<void> {
// no-op: nothing to release
}
}
async function metricsByName(
reader: TestMetricReader,
): Promise<Map<string, MetricData>> {
const collected = await reader.collect();
const result = new Map<string, MetricData>();
for (const scope of collected.resourceMetrics.scopeMetrics) {
for (const metric of scope.metrics) {
result.set(metric.descriptor.name, metric);
}
}
return result;
}
describe("OpenTelemetry metrics bridge", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => tmpDir.removeCallback());
it("snapshot is safe to call regardless of install state", () => {
expect(Array.isArray(snapshotLancedbMetrics())).toBe(true);
});
it("exports object store metrics via observable instruments", async () => {
const reader = new TestMetricReader();
const provider = new MeterProvider({ readers: [reader] });
expect(instrumentLanceDbMetrics(provider)).toBe(true);
// Generate object store activity on the local filesystem (scheme "file").
const db = await connect(tmpDir.name);
const data = Array.from({ length: 256 }, (_, i) => ({ id: i }));
const table = await db.createTable("t", data);
expect(await table.countRows()).toBe(256);
const metrics = await metricsByName(reader);
const requests = metrics.get("lance_object_store_requests_total");
expect(requests).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const requestPoints = (requests!.dataPoints as any[]) ?? [];
expect(requestPoints.length).toBeGreaterThan(0);
for (const p of requestPoints) {
// Labelled by `operation` and `base` (the store scheme by default).
expect(p.attributes).toHaveProperty("base");
expect(p.attributes).toHaveProperty("operation");
}
const totalRequests = requestPoints.reduce((acc, p) => acc + p.value, 0);
expect(totalRequests).toBeGreaterThan(0);
// Histograms are decomposed into bucket / count / sum observable counters.
const bucket = metrics.get(
"lance_object_store_request_duration_seconds_bucket",
);
expect(bucket).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const bucketPoints = (bucket!.dataPoints as any[]) ?? [];
expect(bucketPoints.length).toBeGreaterThan(0);
expect(bucketPoints.every((p) => "le" in p.attributes)).toBe(true);
// The implicit +Inf bucket must be present.
expect(bucketPoints.some((p) => p.attributes.le === "+Inf")).toBe(true);
const count = metrics.get(
"lance_object_store_request_duration_seconds_count",
);
expect(count).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const countPoints = (count!.dataPoints as any[]) ?? [];
expect(countPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
const sum = metrics.get("lance_object_store_request_duration_seconds_sum");
expect(sum).toBeDefined();
// biome-ignore lint/suspicious/noExplicitAny: SDK point shape
const sumPoints = (sum!.dataPoints as any[]) ?? [];
expect(sumPoints.reduce((acc, p) => acc + p.value, 0)).toBeGreaterThan(0);
// Unit handling: only `_sum` keeps the histogram's unit (seconds); `_bucket`
// and `_count` observe cumulative counts and are unitless.
expect(sum!.descriptor.unit).toBe("s");
expect(bucket!.descriptor.unit).toBe("");
expect(count!.descriptor.unit).toBe("");
await provider.shutdown();
});
});
-50
View File
@@ -2992,56 +2992,6 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
}),
).rejects.toThrow();
});
it("reads back the installed spec via getLsmWriteSpec", async () => {
const conn = await connect(tmpDir.name);
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
// Nothing installed yet.
expect(await table.getLsmWriteSpec()).toBeUndefined();
// A real scalar index is needed to name it as a maintained index.
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
await table.createIndex("id");
const indexName = (await table.listIndices())[0].name;
// Bucket spec round-trips, including maintained indexes and writer config
// defaults. Lance writer-config keys are canonically snake_case.
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
const writerConfigDefaults = { durable_write: "false" };
await table.setLsmWriteSpec({
specType: "bucket",
column: "id",
numBuckets: 4,
maintainedIndexes: [indexName],
writerConfigDefaults,
});
const spec = await table.getLsmWriteSpec();
expect(spec).toBeDefined();
expect(spec?.specType).toBe("bucket");
expect(spec?.column).toBe("id");
expect(spec?.numBuckets).toBe(4);
expect(spec?.maintainedIndexes).toEqual([indexName]);
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
// After unset, undefined again.
await table.unsetLsmWriteSpec();
expect(await table.getLsmWriteSpec()).toBeUndefined();
// Identity round-trips (column recovered from the schema).
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
const identity = await table.getLsmWriteSpec();
expect(identity?.specType).toBe("identity");
expect(identity?.column).toBe("id");
await table.unsetLsmWriteSpec();
// Unsharded round-trips (no routing column).
await table.setLsmWriteSpec({ specType: "unsharded" });
const unsharded = await table.getLsmWriteSpec();
expect(unsharded?.specType).toBe("unsharded");
expect(unsharded?.column).toBeFalsy();
});
});
describe("LSM merge insert", () => {
-5
View File
@@ -20,11 +20,6 @@ import { HeaderProvider } from "./header";
// Re-export native header provider for use with connectWithHeaderProvider
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
// underlying recorder/catalog/snapshot functions remain internal plumbing that
// `otel.ts` consumes from the native module.
export { instrumentLanceDbMetrics } from "./otel";
export {
AddColumnsSql,
ConnectionOptions,
-137
View File
@@ -1,137 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import {
type Attributes,
type MeterProvider,
type ObservableResult,
metrics,
} from "@opentelemetry/api";
import {
lancedbMetricsCatalog,
registerLancedbMetricsRecorder,
snapshotLancedbMetrics,
} from "./native";
let instrumented = false;
/**
* Register LanceDB metrics as OpenTelemetry observable instruments.
*
* Installs a process-global metrics recorder and creates one observable
* instrument per LanceDB metric (currently object store request counts, bytes,
* latency, errors, and throttles) on the given (or global) `MeterProvider`. The
* configured `MetricReader` then collects them on its own schedule.
*
* Counters and gauges map directly to observable counters/gauges. Because
* OpenTelemetry has no asynchronous histogram instrument, each histogram is
* exported Prometheus-style as cumulative `le` bucket counts (`<name>_bucket`,
* with an `le` attribute) plus `<name>_count` and `<name>_sum`.
*
* Requires `@opentelemetry/api` (a dependency) and, to actually export, an
* OpenTelemetry SDK such as `@opentelemetry/sdk-metrics`.
*
* @param meterProvider The provider to register instruments on. Defaults to the
* global provider from `@opentelemetry/api`.
* @returns `true` if the recorder is installed and instruments are registered.
* `false` if a different `metrics` recorder is already installed in this
* process (only one global recorder is permitted), in which case a warning is
* emitted and no instruments are created. Calling this more than once is safe;
* instruments are created only on the first successful call.
*/
export function instrumentLanceDbMetrics(
meterProvider?: MeterProvider,
): boolean {
if (!registerLancedbMetricsRecorder()) {
console.warn(
"Could not install the LanceDB metrics recorder: another `metrics` " +
"recorder is already installed in this process. LanceDB metrics will " +
"not be exported via OpenTelemetry.",
);
return false;
}
if (instrumented) {
return true;
}
const provider = meterProvider ?? metrics.getMeterProvider();
const meter = provider.getMeter("lancedb");
const scalarCallback = (metricName: string) => (result: ObservableResult) => {
for (const point of snapshotLancedbMetrics()) {
if (point.name === metricName && point.value != null) {
result.observe(point.value, point.attributes);
}
}
};
const bucketCallback = (metricName: string) => (result: ObservableResult) => {
for (const point of snapshotLancedbMetrics()) {
if (point.name !== metricName || point.buckets == null) {
continue;
}
for (const bucket of point.buckets) {
const attributes: Attributes = {
...point.attributes,
le: bucket.le,
};
result.observe(bucket.cumulativeCount, attributes);
}
}
};
const fieldCallback =
(metricName: string, field: "count" | "sum") =>
(result: ObservableResult) => {
for (const point of snapshotLancedbMetrics()) {
if (point.name !== metricName) {
continue;
}
const value = point[field];
if (value != null) {
result.observe(value, point.attributes);
}
}
};
for (const desc of lancedbMetricsCatalog()) {
const unit = desc.unit ?? "";
if (desc.kind === "counter") {
const counter = meter.createObservableCounter(desc.name, {
unit,
description: desc.description,
});
counter.addCallback(scalarCallback(desc.name));
} else if (desc.kind === "gauge") {
const gauge = meter.createObservableGauge(desc.name, {
unit,
description: desc.description,
});
gauge.addCallback(scalarCallback(desc.name));
} else if (desc.kind === "histogram") {
// `_bucket` and `_count` observe cumulative sample counts, not the
// histogram's measured quantity, so they are unitless; only `_sum`
// carries the histogram's unit.
const bucket = meter.createObservableCounter(`${desc.name}_bucket`, {
description: `${desc.description} (cumulative buckets)`,
});
bucket.addCallback(bucketCallback(desc.name));
const count = meter.createObservableCounter(`${desc.name}_count`, {
description: `${desc.description} (count)`,
});
count.addCallback(fieldCallback(desc.name, "count"));
const sum = meter.createObservableCounter(`${desc.name}_sum`, {
unit,
description: `${desc.description} (sum)`,
});
sum.addCallback(fieldCallback(desc.name, "sum"));
}
}
instrumented = true;
return true;
}
-20
View File
@@ -585,17 +585,6 @@ export abstract class Table {
* @returns {Promise<void>}
*/
abstract unsetLsmWriteSpec(): Promise<void>;
/**
* Read the {@link LsmWriteSpec} currently installed on this table.
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* The returned spec — including its `maintainedIndexes` and
* `writerConfigDefaults` — mirrors what was passed to
* {@link Table#setLsmWriteSpec}.
* @returns {Promise<LsmWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
/**
* Drain and close any cached MemWAL shard writers held for this table.
*
@@ -1102,15 +1091,6 @@ export class LocalTable extends Table {
return await this.inner.unsetLsmWriteSpec();
}
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
// The native binding types `specType` as a plain `string`; narrow it back
// to the public union. The Rust `From` impl only ever emits one of the
// three valid values, so the cast is safe.
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
| LsmWriteSpec
| undefined;
}
async closeLsmWriters(): Promise<void> {
return await this.inner.closeLsmWriters();
}
-2
View File
@@ -44,7 +44,6 @@
"@biomejs/biome": "^1.7.3",
"@jest/globals": "^29.7.0",
"@napi-rs/cli": "3.7.0",
"@opentelemetry/sdk-metrics": "^1.30.0",
"@types/axios": "^0.14.0",
"@types/jest": "^29.1.2",
"@types/node": "22.7.4",
@@ -93,7 +92,6 @@
"version": "napi version"
},
"dependencies": {
"@opentelemetry/api": "^1.9.0",
"reflect-metadata": "^0.2.2"
},
"optionalDependencies": {
-53
View File
@@ -8,9 +8,6 @@ importers:
.:
dependencies:
'@opentelemetry/api':
specifier: ^1.9.0
version: 1.9.1
apache-arrow:
specifier: '>=15.0.0 <=18.1.0'
version: 18.1.0
@@ -36,9 +33,6 @@ importers:
'@napi-rs/cli':
specifier: 3.7.0
version: 3.7.0(@emnapi/core@1.10.0)(@emnapi/runtime@1.10.0)(@types/node@22.7.4)
'@opentelemetry/sdk-metrics':
specifier: ^1.30.0
version: 1.30.1(@opentelemetry/api@1.9.1)
'@types/axios':
specifier: ^0.14.0
version: 0.14.4
@@ -1313,32 +1307,6 @@ packages:
'@octokit/types@16.0.0':
resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==}
'@opentelemetry/api@1.9.1':
resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}
engines: {node: '>=8.0.0'}
'@opentelemetry/core@1.30.1':
resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': '>=1.0.0 <1.10.0'
'@opentelemetry/resources@1.30.1':
resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': '>=1.0.0 <1.10.0'
'@opentelemetry/sdk-metrics@1.30.1':
resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==}
engines: {node: '>=14'}
peerDependencies:
'@opentelemetry/api': '>=1.3.0 <1.10.0'
'@opentelemetry/semantic-conventions@1.28.0':
resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==}
engines: {node: '>=14'}
'@protobufjs/aspromise@1.1.2':
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
@@ -4957,27 +4925,6 @@ snapshots:
dependencies:
'@octokit/openapi-types': 27.0.0
'@opentelemetry/api@1.9.1': {}
'@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.1)':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/semantic-conventions': 1.28.0
'@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.1)':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
'@opentelemetry/semantic-conventions': 1.28.0
'@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.1)':
dependencies:
'@opentelemetry/api': 1.9.1
'@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.1)
'@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.1)
'@opentelemetry/semantic-conventions@1.28.0': {}
'@protobufjs/aspromise@1.1.2':
optional: true
-1
View File
@@ -12,7 +12,6 @@ mod header;
mod index;
mod iterator;
pub mod merge;
pub mod otel;
pub mod permutation;
mod query;
pub mod remote;
-119
View File
@@ -1,119 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Node.js bindings over [`lancedb::metrics_otel`].
//!
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
//! core crate; this module only converts the core snapshot types into napi
//! objects and exposes the three entry points to JavaScript, where
//! `lancedb/otel.ts` bridges them into the user's OpenTelemetry `MeterProvider`.
use std::collections::HashMap;
use lancedb::metrics_otel::{MetricPoint as CoreMetricPoint, MetricValue};
use napi_derive::napi;
/// One cumulative histogram bucket: all samples with value `<= le`.
#[napi(object)]
pub struct MetricBucket {
/// The inclusive upper bound of the bucket, or `"+Inf"` for the final bucket.
pub le: String,
/// Cumulative number of samples less than or equal to `le`.
pub cumulative_count: f64,
}
/// One aggregated metric data point. For counters and gauges only `value` is
/// set; for histograms `buckets` (cumulative `le` counts), `count`, and `sum`
/// are set.
#[napi(object)]
pub struct MetricPoint {
pub name: String,
pub kind: String,
pub attributes: HashMap<String, String>,
pub value: Option<f64>,
pub buckets: Option<Vec<MetricBucket>>,
pub count: Option<f64>,
pub sum: Option<f64>,
}
impl From<CoreMetricPoint> for MetricPoint {
fn from(point: CoreMetricPoint) -> Self {
let kind = point.kind.as_str().to_string();
let (value, buckets, count, sum) = match point.value {
MetricValue::Scalar(v) => (Some(v), None, None, None),
MetricValue::Histogram {
buckets,
count,
sum,
} => (
None,
Some(
buckets
.into_iter()
// Counts stay well within the f64-exact integer range
// (2^53), so this cast is lossless in practice and keeps
// the values plain JS numbers for OpenTelemetry.
.map(|(le, cumulative_count)| MetricBucket {
le,
cumulative_count: cumulative_count as f64,
})
.collect(),
),
Some(count as f64),
Some(sum),
),
};
Self {
name: point.name,
kind,
attributes: point.attributes,
value,
buckets,
count,
sum,
}
}
}
/// A described metric, used by the JavaScript layer to create instruments up front.
#[napi(object)]
pub struct MetricDescription {
pub name: String,
pub kind: String,
pub unit: Option<String>,
pub description: String,
}
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
///
/// Returns `true` if the recorder is installed (now or previously). Returns
/// `false` if a *different* recorder is already installed — `metrics` allows
/// only one global recorder per process, so LanceDB cannot coexist with another.
#[napi]
pub fn register_lancedb_metrics_recorder() -> bool {
lancedb::metrics_otel::register_metrics_recorder()
}
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
#[napi]
pub fn lancedb_metrics_catalog() -> Vec<MetricDescription> {
lancedb::metrics_otel::metrics_catalog()
.into_iter()
.map(|desc| MetricDescription {
name: desc.name,
kind: desc.kind.as_str().to_string(),
unit: desc.unit,
description: desc.description,
})
.collect()
}
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
/// is installed.
#[napi]
pub fn snapshot_lancedb_metrics() -> Vec<MetricPoint> {
lancedb::metrics_otel::snapshot_metrics()
.into_iter()
.map(MetricPoint::from)
.collect()
}
-51
View File
@@ -411,16 +411,6 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
let spec = self
.inner_ref()?
.get_lsm_write_spec()
.await
.default_error()?;
Ok(spec.map(LsmWriteSpec::from))
}
#[napi(catch_unwind)]
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
self.inner_ref()?.close_lsm_writers().await.default_error()
@@ -738,47 +728,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
use lancedb::table::LsmWriteSpec as Native;
match spec {
Native::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "bucket".to_string(),
column: Some(column),
num_buckets: Some(num_buckets),
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Identity {
column,
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "identity".to_string(),
column: Some(column),
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Unsharded {
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "unsharded".to_string(),
column: None,
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
}
}
}
/// Statistics about a compaction operation.
#[napi(object)]
#[derive(Clone, Debug)]
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.35.0-beta.0"
current_version = "0.34.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.35.0-beta.0"
version = "0.34.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
@@ -47,6 +47,6 @@ pyo3-build-config = { version = "0.28", features = [
] }
[features]
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
fp16kernels = ["lancedb/fp16kernels"]
remote = ["lancedb/remote"]
-5
View File
@@ -47,10 +47,6 @@ repository = "https://github.com/lancedb/lancedb"
pylance = [
"pylance>=5.0.0b5",
]
# A library only needs the OpenTelemetry API; the application supplies and
# configures the SDK (the actual exporter/reader). See
# https://opentelemetry.io/docs/languages/python/instrumentation/
otel = ["opentelemetry-api"]
tests = [
"aiohttp>=3.9.0",
"boto3>=1.28.57",
@@ -65,7 +61,6 @@ tests = [
"pylance>=5.0.0b5",
"requests>=2.31.0",
"datafusion>=52,<53",
"opentelemetry-sdk>=1.30.0",
]
dev = [
"ruff>=0.3.0",
+2 -8
View File
@@ -149,14 +149,8 @@ def connect(
For object storage, use a URI prefix:
>>> db = lancedb.connect( # doctest: +SKIP
... "s3://my-bucket/lancedb",
... storage_options={
... "aws_access_key_id": "***",
... "aws_secret_access_key": "***",
... "aws_region": "us-east-1",
... },
... )
>>> db = lancedb.connect("s3://my-bucket/lancedb",
... storage_options={"aws_access_key_id": "***"})
For tests and temporary data, use an in-memory database:
-20
View File
@@ -30,25 +30,6 @@ IvfHnswPq: type[HnswPq] = HnswPq
IvfHnswSq: type[HnswSq] = HnswSq
IvfHnswFlat: type[HnswFlat] = HnswFlat
class MetricPoint:
name: str
kind: str
attributes: Dict[str, str]
value: Optional[float]
buckets: Optional[List[Tuple[str, int]]]
count: Optional[int]
sum: Optional[float]
class MetricDescription:
name: str
kind: str
unit: Optional[str]
description: str
def register_lancedb_metrics_recorder() -> bool: ...
def lancedb_metrics_catalog() -> List[MetricDescription]: ...
def snapshot_lancedb_metrics() -> List[MetricPoint]: ...
class PyExpr:
"""A type-safe DataFusion expression node (Rust-side handle)."""
@@ -248,7 +229,6 @@ class Table:
async def set_unenforced_primary_key(self, columns: List[str]) -> None: ...
async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
async def unset_lsm_write_spec(self) -> None: ...
async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
async def close_lsm_writers(self) -> None: ...
@property
def tags(self) -> Tags: ...
+1 -11
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
from functools import cached_property
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Union
from typing import TYPE_CHECKING, List, Optional, Sequence, Union
import numpy as np
@@ -56,16 +56,6 @@ class OllamaEmbeddings(TextEmbeddingFunction):
embeddings = self._compute_embedding(texts)
return list(embeddings)
def __getstate__(self) -> dict[str, Any]:
state = super().__getstate__()
state["__dict__"] = {
k: v for k, v in state["__dict__"].items() if k != "_ollama_client"
}
return state
def __setstate__(self, state: dict[str, Any]) -> None:
super().__setstate__(state)
@cached_property
def _ollama_client(self) -> "ollama.Client":
ollama = attempt_import_or_raise("ollama")
-170
View File
@@ -1,170 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Bridge LanceDB's internal metrics into OpenTelemetry.
LanceDB (through Lance core) publishes metrics (currently object store request
counts, bytes, latency, errors, and throttles) through the Rust ``metrics``
facade. This module installs a process-global recorder that aggregates them and
registers OpenTelemetry observable instruments that report the aggregated values
into the user's ``MeterProvider``.
The bridge is generic: every metric LanceDB describes is surfaced automatically,
with no per-metric Python code. Histograms have no asynchronous OpenTelemetry
instrument, so each is exported Prometheus-style as cumulative ``le`` buckets
plus ``_count`` and ``_sum`` observable counters.
"""
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Optional
from ._lancedb import (
lancedb_metrics_catalog,
register_lancedb_metrics_recorder,
snapshot_lancedb_metrics,
)
if TYPE_CHECKING:
from opentelemetry.metrics import MeterProvider
_INSTRUMENTED = False
def instrument_lancedb_metrics(
meter_provider: Optional["MeterProvider"] = None,
) -> bool:
"""Register LanceDB metrics as OpenTelemetry observable instruments.
Installs a process-global metrics recorder and creates one observable
instrument per LanceDB metric on the given (or global) ``MeterProvider``. The
user's configured ``MetricReader`` then collects them on its own schedule.
Counters and gauges map directly to observable counters/gauges. Each
histogram is exported as cumulative ``le`` bucket counts (``<name>_bucket``,
with an ``le`` attribute) plus ``<name>_count`` and ``<name>_sum``.
Parameters
----------
meter_provider : opentelemetry.metrics.MeterProvider, optional
The provider to register instruments on. Defaults to the global provider
from ``opentelemetry.metrics.get_meter_provider()``.
Returns
-------
bool
``True`` if the recorder is installed and instruments are registered.
``False`` if a different ``metrics`` recorder is already installed in
this process (``metrics`` permits only one global recorder), in which
case a warning is emitted and no instruments are created.
Notes
-----
Requires the OpenTelemetry API (``pip install lancedb[otel]``) and, to
actually export, an OpenTelemetry SDK (``pip install opentelemetry-sdk``)
configured by the application. Calling this more than once is safe;
instruments are created only on the first successful call.
"""
global _INSTRUMENTED
try:
from opentelemetry.metrics import Observation, get_meter_provider
except ImportError as exc:
raise ImportError(
"instrument_lancedb_metrics requires the OpenTelemetry API/SDK. "
"Install it with `pip install lancedb[otel]` or "
"`pip install opentelemetry-sdk`."
) from exc
if not register_lancedb_metrics_recorder():
warnings.warn(
"Could not install the LanceDB metrics recorder: another `metrics` "
"recorder is already installed in this process. LanceDB metrics will "
"not be exported via OpenTelemetry.",
stacklevel=2,
)
return False
if _INSTRUMENTED:
return True
provider = meter_provider or get_meter_provider()
meter = provider.get_meter("lancedb")
def scalar_callback(metric_name: str):
def callback(_options):
return [
Observation(point.value, point.attributes)
for point in snapshot_lancedb_metrics()
if point.name == metric_name and point.value is not None
]
return callback
def bucket_callback(metric_name: str):
def callback(_options):
observations = []
for point in snapshot_lancedb_metrics():
if point.name != metric_name or point.buckets is None:
continue
for le, cumulative in point.buckets:
attributes = dict(point.attributes)
attributes["le"] = le
observations.append(Observation(cumulative, attributes))
return observations
return callback
def field_callback(metric_name: str, field: str):
def callback(_options):
observations = []
for point in snapshot_lancedb_metrics():
if point.name != metric_name:
continue
value = getattr(point, field)
if value is not None:
observations.append(Observation(value, point.attributes))
return observations
return callback
for desc in lancedb_metrics_catalog():
unit = desc.unit or ""
if desc.kind == "counter":
meter.create_observable_counter(
desc.name,
callbacks=[scalar_callback(desc.name)],
unit=unit,
description=desc.description,
)
elif desc.kind == "gauge":
meter.create_observable_gauge(
desc.name,
callbacks=[scalar_callback(desc.name)],
unit=unit,
description=desc.description,
)
elif desc.kind == "histogram":
# `_bucket` and `_count` observe cumulative sample counts, not the
# histogram's measured quantity, so they are unitless; only `_sum`
# carries the histogram's unit.
meter.create_observable_counter(
f"{desc.name}_bucket",
callbacks=[bucket_callback(desc.name)],
description=f"{desc.description} (cumulative buckets)",
)
meter.create_observable_counter(
f"{desc.name}_count",
callbacks=[field_callback(desc.name, "count")],
description=f"{desc.description} (count)",
)
meter.create_observable_counter(
f"{desc.name}_sum",
callbacks=[field_callback(desc.name, "sum")],
unit=unit,
description=f"{desc.description} (sum)",
)
_INSTRUMENTED = True
return True
+4 -11
View File
@@ -11,7 +11,7 @@ import pyarrow as pa
from ._lancedb import async_permutation_builder, PermutationReader
from .table import LanceTable, Table
from .background_loop import LOOP
from .util import batch_to_tensor, batch_to_tensor_dict, batch_to_tensor_rows
from .util import batch_to_tensor, batch_to_tensor_rows
from typing import Any, Callable, Iterator, Literal, Optional, TYPE_CHECKING, Union
if TYPE_CHECKING:
@@ -946,7 +946,6 @@ class Permutation:
"pandas",
"arrow",
"torch",
"torch_row",
"torch_col",
"polars",
],
@@ -962,19 +961,15 @@ class Permutation:
- "python_col" - the batch will be a dict of lists (one entry per column)
- "pandas" - the batch will be a pandas DataFrame
- "arrow" - the batch will be a pyarrow RecordBatch
- "torch" - the batch will be a list of per-row dicts mapping column
name to a 0-D torch tensor. Works with the default
``torch.utils.data.DataLoader`` collate, which stacks the per-row
dicts back into a dict of batched tensors.
- "torch_row" - the batch will be a list of tensors, one per row
- "torch" - the batch will be a list of tensors, one per row
- "torch_col" - the batch will be a 2D torch tensor (first dim indexes columns)
- "polars" - the batch will be a polars DataFrame
Conversion may or may not involve a data copy. Lance uses Arrow internally
and so it is able to zero-copy to the arrow and polars formats.
Conversion to torch and torch_col will be zero-copy but will only support a
subset of data types (numeric types).
Conversion to torch_col will be zero-copy but will only support a subset of data
types (numeric types).
Conversion to numpy and/or pandas will typically be zero-copy for numeric
types. Conversion of strings, lists, and structs will require creating python
@@ -995,8 +990,6 @@ class Permutation:
elif format == "arrow":
return self.with_transform(Transforms.arrow2arrow)
elif format == "torch":
return self.with_transform(batch_to_tensor_dict)
elif format == "torch_row":
return self.with_transform(batch_to_tensor_rows)
elif format == "torch_col":
return self.with_transform(batch_to_tensor)
-4
View File
@@ -912,10 +912,6 @@ class RemoteTable(Table):
"""Not supported on LanceDB Cloud."""
return LOOP.run(self._table.unset_lsm_write_spec())
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the installed LsmWriteSpec, or ``None``."""
return LOOP.run(self._table.get_lsm_write_spec())
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
-16
View File
@@ -3759,11 +3759,6 @@ class LanceTable(Table):
[`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec]."""
return LOOP.run(self._table.unset_lsm_write_spec())
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the installed LsmWriteSpec, or ``None``. See
[`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
return LOOP.run(self._table.get_lsm_write_spec())
def close_lsm_writers(self) -> None:
"""Close cached MemWAL shard writers. See
[`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
@@ -4422,17 +4417,6 @@ class AsyncTable:
"""
await self._inner.unset_lsm_write_spec()
async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the LsmWriteSpec currently installed on this table.
Returns ``None`` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with `unset_lsm_write_spec`).
The returned spec — including its ``maintained_indexes`` and
``writer_config_defaults`` — mirrors what was passed to
`set_lsm_write_spec`.
"""
return await self._inner.get_lsm_write_spec()
async def close_lsm_writers(self) -> None:
"""Drain and close any cached MemWAL shard writers for this table.
+1 -39
View File
@@ -177,10 +177,7 @@ def flatten_columns(tbl: pa.Table, flatten: Optional[Union[int, bool]] = None):
continue
else:
break
# `bool` is a subclass of `int`, so guard against it explicitly: `flatten=False`
# (and `None`) must mean "do not flatten" rather than falling into the integer
# branch and raising on the `flatten <= 0` check.
elif isinstance(flatten, int) and not isinstance(flatten, bool):
elif isinstance(flatten, int):
if flatten <= 0:
raise ValueError(
"Please specify a positive integer for flatten or the boolean "
@@ -518,38 +515,3 @@ def batch_to_tensor_rows(batch: pa.RecordBatch):
stacked = torch.tensor(numpy.column_stack(columns))
rows = list(stacked.unbind(dim=0))
return rows
def batch_to_tensor_dict(batch: pa.RecordBatch):
"""
Convert a PyArrow RecordBatch to a list of per-row dicts of PyTorch Tensors.
Each column is first converted to a 1-D tensor (zero-copy via DLPack), then
sliced per row. The result is a list whose length is ``batch.num_rows`` and
whose items are dicts keyed by column name. This shape composes directly
with PyTorch's default ``DataLoader`` collate, which stacks the per-row
dicts back into a dict of batched tensors.
Fails if torch is not installed.
Fails if a column's data type is not supported by PyTorch.
Parameters
----------
batch : pa.RecordBatch
The record batch to convert.
Returns
-------
list[dict[str, torch.Tensor]]
One per-row dict per row in the batch. Each dict maps column name to a
0-D tensor view into the column.
"""
torch = attempt_import_or_raise("torch", "torch")
tensors = {
name: torch.from_dlpack(col)
for name, col in zip(batch.schema.names, batch.columns)
}
return [
{name: tensor[i] for name, tensor in tensors.items()}
for i in range(batch.num_rows)
]
-17
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
import pickle
from typing import List, Optional, Union
from unittest.mock import MagicMock, patch
@@ -592,22 +591,6 @@ def test_openai_no_retry_on_401(mock_sleep):
assert mock_sleep.call_count == 0
def test_ollama_embeddings_pickle():
"""OllamaEmbeddings must pickle even after the cached client is created."""
registry = get_registry()
model = registry.get("ollama").create(name="nomic-embed-text")
# Simulate accessing the cached client, which stores it on the instance.
model.__dict__["_ollama_client"] = MagicMock()
pickled = pickle.dumps(model)
restored = pickle.loads(pickled)
assert restored.name == "nomic-embed-text"
assert restored.host == "http://localhost:11434"
assert "_ollama_client" not in restored.__dict__
def test_url_retrieve_downloads_image():
"""
Embedding functions like open-clip, siglip, and jinaai use url_retrieve()
@@ -11,7 +11,6 @@ import lancedb
import pyarrow as pa
import pytest
from lancedb._lancedb import LsmWriteSpec
from lancedb.index import BTree
SCHEMA = pa.schema(
[
@@ -137,76 +136,3 @@ def test_lsm_write_spec_identity_and_writer_config_defaults():
s = s.with_writer_config_defaults({"durable_write": "false"})
assert s.writer_config_defaults == {"durable_write": "false"}
assert "durable_write" in repr(s)
def test_get_lsm_write_spec(tmp_path):
_db, table = _make_table(tmp_path)
table.set_unenforced_primary_key("id")
# None when nothing is installed.
assert table.get_lsm_write_spec() is None
# A real scalar index is needed to name it as a maintained index.
table.create_index("id", config=BTree())
idx_name = table.list_indices()[0].name
# Bucket spec round-trips, including maintained indexes and writer config
# defaults.
table.set_lsm_write_spec(
LsmWriteSpec.bucket("id", 4)
.with_maintained_indexes([idx_name])
.with_writer_config_defaults({"durable_write": "false"})
)
spec = table.get_lsm_write_spec()
assert spec is not None
assert spec.spec_type == "bucket"
assert spec.column == "id"
assert spec.num_buckets == 4
assert spec.maintained_indexes == [idx_name]
assert spec.writer_config_defaults == {"durable_write": "false"}
# After unset, None again.
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema).
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column).
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
assert spec.column is None
@pytest.mark.asyncio
async def test_async_get_lsm_write_spec(tmp_path):
db = await lancedb.connect_async(
tmp_path, read_consistency_interval=timedelta(seconds=0)
)
table = await db.create_table(
"t",
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
)
assert await table.get_lsm_write_spec() is None
# A real scalar index is needed to name it as a maintained index.
await table.create_index("id", config=BTree())
idx_name = (await table.list_indices())[0].name
await table.set_lsm_write_spec(
LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
)
spec = await table.get_lsm_write_spec()
assert spec is not None
assert spec.spec_type == "bucket"
assert spec.column == "id"
assert spec.num_buckets == 8
assert spec.maintained_indexes == [idx_name]
await table.unset_lsm_write_spec()
assert await table.get_lsm_write_spec() is None
+7 -34
View File
@@ -935,41 +935,14 @@ def test_transform_fn(mem_db):
try:
import torch
# "torch" returns a list of per-row dicts. Default DataLoader collate
# stacks the per-row dicts back into a dict of batched tensors.
torch_perm = permutation.with_format("torch")
torch_batch = list(torch_perm.iter(10, skip_last_batch=False))[0]
assert isinstance(torch_batch, list)
assert len(torch_batch) == 10
assert isinstance(torch_batch[0], dict)
assert set(torch_batch[0].keys()) == {"id", "value"}
assert isinstance(torch_batch[0]["id"], torch.Tensor)
assert torch_batch[0]["id"].dtype == torch.int64
rows = torch_perm.__getitems__([0, 1, 2])
assert isinstance(rows, list)
assert len(rows) == 3
assert isinstance(rows[0], dict)
assert set(rows[0].keys()) == {"id", "value"}
assert isinstance(rows[0]["id"], torch.Tensor)
# "torch_row" returns a list of tensors, one per row.
torch_rows = list(
permutation.with_format("torch_row").iter(10, skip_last_batch=False)
torch_result = list(
permutation.with_format("torch").iter(10, skip_last_batch=False)
)[0]
assert isinstance(torch_rows, list)
assert len(torch_rows) == 10
assert isinstance(torch_rows[0], torch.Tensor)
assert torch_rows[0].shape == (2,)
assert torch_rows[0].dtype == torch.int64
# "torch_col" stacks columns into a single 2D tensor.
torch_col = list(
permutation.with_format("torch_col").iter(10, skip_last_batch=False)
)[0]
assert isinstance(torch_col, torch.Tensor)
assert torch_col.shape == (2, 10)
assert torch_col.dtype == torch.int64
assert isinstance(torch_result, list)
assert len(torch_result) == 10
assert isinstance(torch_result[0], torch.Tensor)
assert torch_result[0].shape == (2,)
assert torch_result[0].dtype == torch.int64
except ImportError:
# Skip check if torch is not installed
pass
+19 -36
View File
@@ -7,8 +7,7 @@ Tests for S3 bucket names containing dots.
Related issue: https://github.com/lancedb/lancedb/issues/1898
These tests validate the early error checking for S3 bucket names with dots.
When validation succeeds, eager namespace initialization may still fail later
because these tests intentionally do not provide real S3 credentials.
No actual S3 connection is made - validation happens before connection.
"""
import pytest
@@ -21,13 +20,6 @@ BUCKET_WITH_DOTS_AND_AWS_REGION = ("s3://my.bucket.name", {"aws_region": "us-eas
BUCKET_WITHOUT_DOTS = "s3://my-bucket/path"
def assert_not_rejected_for_bucket_dots(connect):
try:
connect()
except ValueError as err:
assert "contains dots" not in str(err)
class TestS3BucketWithDotsSync:
"""Tests for connect()."""
@@ -35,22 +27,19 @@ class TestS3BucketWithDotsSync:
with pytest.raises(ValueError, match="contains dots"):
lancedb.connect(BUCKET_WITH_DOTS)
def test_bucket_with_dots_and_region_is_not_rejected(self):
def test_bucket_with_dots_and_region_passes(self):
uri, opts = BUCKET_WITH_DOTS_AND_REGION
assert_not_rejected_for_bucket_dots(
lambda: lancedb.connect(uri, storage_options=opts)
)
db = lancedb.connect(uri, storage_options=opts)
assert db is not None
def test_bucket_with_dots_and_aws_region_is_not_rejected(self):
def test_bucket_with_dots_and_aws_region_passes(self):
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
assert_not_rejected_for_bucket_dots(
lambda: lancedb.connect(uri, storage_options=opts)
)
db = lancedb.connect(uri, storage_options=opts)
assert db is not None
def test_bucket_without_dots_is_not_rejected(self):
assert_not_rejected_for_bucket_dots(
lambda: lancedb.connect(BUCKET_WITHOUT_DOTS)
)
def test_bucket_without_dots_passes(self):
db = lancedb.connect(BUCKET_WITHOUT_DOTS)
assert db is not None
class TestS3BucketWithDotsAsync:
@@ -62,24 +51,18 @@ class TestS3BucketWithDotsAsync:
await lancedb.connect_async(BUCKET_WITH_DOTS)
@pytest.mark.asyncio
async def test_bucket_with_dots_and_region_is_not_rejected(self):
async def test_bucket_with_dots_and_region_passes(self):
uri, opts = BUCKET_WITH_DOTS_AND_REGION
try:
await lancedb.connect_async(uri, storage_options=opts)
except ValueError as err:
assert "contains dots" not in str(err)
db = await lancedb.connect_async(uri, storage_options=opts)
assert db is not None
@pytest.mark.asyncio
async def test_bucket_with_dots_and_aws_region_is_not_rejected(self):
async def test_bucket_with_dots_and_aws_region_passes(self):
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
try:
await lancedb.connect_async(uri, storage_options=opts)
except ValueError as err:
assert "contains dots" not in str(err)
db = await lancedb.connect_async(uri, storage_options=opts)
assert db is not None
@pytest.mark.asyncio
async def test_bucket_without_dots_is_not_rejected(self):
try:
await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
except ValueError as err:
assert "contains dots" not in str(err)
async def test_bucket_without_dots_passes(self):
db = await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
assert db is not None
+4 -36
View File
@@ -148,47 +148,15 @@ def test_permutation_dataloader(mem_db):
for batch in dataloader:
assert batch["a"].size(0) == 10
# "torch" produces a list of per-row dicts per batch. The default
# DataLoader collate stacks the per-row dicts back into a batched dict.
torch_perm = permutation.with_format("torch")
batch = next(torch_perm.iter(10, skip_last_batch=False))
assert isinstance(batch, list)
assert len(batch) == 10
assert isinstance(batch[0], dict)
assert isinstance(batch[0]["a"], torch.Tensor)
rows = torch_perm.__getitems__([0, 1, 2])
assert isinstance(rows, list)
assert len(rows) == 3
assert isinstance(rows[0], dict)
assert isinstance(rows[0]["a"], torch.Tensor)
dataloader = torch.utils.data.DataLoader(torch_perm, batch_size=10, shuffle=True)
for batch in dataloader:
assert isinstance(batch, dict)
assert batch["a"].shape == (10,)
# Spawn-based workers exercise the pickle round-trip path: the new
# transform-as-list shape must survive pickling so workers produce the
# same per-row dicts the parent does.
spawn_loader = torch.utils.data.DataLoader(
torch_perm,
batch_size=10,
num_workers=2,
multiprocessing_context="spawn",
)
for batch in spawn_loader:
assert isinstance(batch, dict)
assert batch["a"].shape == (10,)
# "torch_row" returns a list of row tensors. Works with the default
# DataLoader collate (stacks rows into 2D).
row_perm = permutation.with_format("torch_row")
dataloader = torch.utils.data.DataLoader(row_perm, batch_size=10, shuffle=True)
permutation = permutation.with_format("torch")
dataloader = torch.utils.data.DataLoader(permutation, batch_size=10, shuffle=True)
for batch in dataloader:
assert batch.size(0) == 10
assert batch.size(1) == 1
col_perm = permutation.with_format("torch_col")
permutation = permutation.with_format("torch_col")
dataloader = torch.utils.data.DataLoader(
col_perm, collate_fn=lambda x: x, batch_size=10, shuffle=True
permutation, collate_fn=lambda x: x, batch_size=10, shuffle=True
)
for batch in dataloader:
assert batch.size(0) == 1
+1 -33
View File
@@ -25,42 +25,10 @@ import pandas as pd
import polars as pl
import pytest
import lancedb
from lancedb.util import flatten_columns, get_uri_scheme, join_uri, value_to_sql
from lancedb.util import get_uri_scheme, join_uri, value_to_sql
from utils import exception_output
def _struct_table() -> pa.Table:
return pa.table(
{
"id": [1, 2],
"nested": pa.array([{"a": 1, "b": 2}, {"a": 3, "b": 4}]),
}
)
def test_flatten_columns():
tbl = _struct_table()
# None / False mean "do not flatten": the struct column is preserved.
# `False` is a regression guard: because bool is a subclass of int it used
# to fall into the integer branch and raise ValueError (see issue).
for no_flatten in (None, False):
result = flatten_columns(tbl, no_flatten)
assert result.column_names == ["id", "nested"]
# True flattens all nested levels.
flattened = flatten_columns(tbl, True)
assert flattened.column_names == ["id", "nested.a", "nested.b"]
# A positive integer flattens up to that depth.
flattened = flatten_columns(tbl, 1)
assert flattened.column_names == ["id", "nested.a", "nested.b"]
# Non-positive integers are still rejected.
with pytest.raises(ValueError):
flatten_columns(tbl, 0)
def test_normalize_uri():
uris = [
"relative/path",
-10
View File
@@ -27,7 +27,6 @@ pub mod header;
pub mod index;
pub mod namespace;
pub mod oauth;
pub mod otel;
pub mod permutation;
pub mod query;
pub mod runtime;
@@ -62,15 +61,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyAsyncPermutationBuilder>()?;
m.add_class::<PyPermutationReader>()?;
m.add_class::<PyExpr>()?;
// OpenTelemetry metrics bridge
m.add_class::<otel::PyMetricPoint>()?;
m.add_class::<otel::PyMetricDescription>()?;
m.add_function(wrap_pyfunction!(
otel::register_lancedb_metrics_recorder,
m
)?)?;
m.add_function(wrap_pyfunction!(otel::lancedb_metrics_catalog, m)?)?;
m.add_function(wrap_pyfunction!(otel::snapshot_lancedb_metrics, m)?)?;
m.add_function(wrap_pyfunction!(connect, m)?)?;
m.add_function(wrap_pyfunction!(connect_namespace, m)?)?;
m.add_function(wrap_pyfunction!(connect_namespace_client, m)?)?;
-96
View File
@@ -1,96 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Python-facing wrappers over [`lancedb::metrics_otel`].
//!
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
//! core crate; this module only converts the core snapshot types into PyO3
//! classes and exposes the three entry points to Python, where
//! `lancedb/otel.py` bridges them into the user's OpenTelemetry `MeterProvider`.
use std::collections::HashMap;
use lancedb::metrics_otel::{MetricPoint, MetricValue};
use pyo3::prelude::*;
/// One metric data point exposed to Python. For counters and gauges only
/// `value` is set; for histograms `buckets` (cumulative `le` counts), `count`,
/// and `sum` are set.
#[pyclass(name = "MetricPoint", get_all)]
pub struct PyMetricPoint {
name: String,
kind: String,
attributes: HashMap<String, String>,
value: Option<f64>,
buckets: Option<Vec<(String, u64)>>,
count: Option<u64>,
sum: Option<f64>,
}
impl From<MetricPoint> for PyMetricPoint {
fn from(point: MetricPoint) -> Self {
let kind = point.kind.as_str().to_string();
let (value, buckets, count, sum) = match point.value {
MetricValue::Scalar(v) => (Some(v), None, None, None),
MetricValue::Histogram {
buckets,
count,
sum,
} => (None, Some(buckets), Some(count), Some(sum)),
};
Self {
name: point.name,
kind,
attributes: point.attributes,
value,
buckets,
count,
sum,
}
}
}
/// A described metric, used by the Python layer to create instruments up front.
#[pyclass(name = "MetricDescription", get_all)]
pub struct PyMetricDescription {
name: String,
kind: String,
unit: Option<String>,
description: String,
}
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
///
/// Returns `True` if the recorder is installed (now or previously). Returns
/// `False` if a *different* recorder is already installed — `metrics` allows
/// only one global recorder per process, so LanceDB cannot coexist with another.
#[pyfunction]
pub fn register_lancedb_metrics_recorder() -> bool {
lancedb::metrics_otel::register_metrics_recorder()
}
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
#[pyfunction]
pub fn lancedb_metrics_catalog() -> Vec<PyMetricDescription> {
lancedb::metrics_otel::metrics_catalog()
.into_iter()
.map(|desc| PyMetricDescription {
name: desc.name,
kind: desc.kind.as_str().to_string(),
unit: desc.unit,
description: desc.description,
})
.collect()
}
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
/// is installed.
///
/// The read is lock-free but not O(1): it walks every registered series and
/// allocates owned copies of their names and labels. The GIL is released across
/// that work so a periodic collection doesn't stall other Python threads.
#[pyfunction]
pub fn snapshot_lancedb_metrics(py: Python<'_>) -> Vec<PyMetricPoint> {
let points = py.detach(lancedb::metrics_otel::snapshot_metrics);
points.into_iter().map(PyMetricPoint::from).collect()
}
-14
View File
@@ -322,12 +322,6 @@ impl From<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(inner: lancedb::table::LsmWriteSpec) -> Self {
Self { inner }
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct AddColumnsResult {
@@ -1035,14 +1029,6 @@ impl Table {
})
}
pub fn get_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let spec = inner.get_lsm_write_spec().await.infer_error()?;
Ok(spec.map(LsmWriteSpec::from))
})
}
pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
-101
View File
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pyarrow as pa
import pytest
# The metrics recorder is process-global and installed once, so the whole
# bridge is exercised in a single test to avoid cross-test global-state coupling.
def _metrics_by_name(reader):
data = reader.get_metrics_data()
result = {}
for resource_metrics in data.resource_metrics:
for scope_metrics in resource_metrics.scope_metrics:
for metric in scope_metrics.metrics:
result[metric.name] = metric
return result
def test_instrument_lancedb_metrics_exports_object_store_metrics(tmp_path):
pytest.importorskip("opentelemetry.sdk.metrics")
from lancedb.otel import instrument_lancedb_metrics
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader])
assert instrument_lancedb_metrics(provider)
# The catalog is populated once the recorder is installed.
from lancedb._lancedb import lancedb_metrics_catalog
catalog = {desc.name: desc for desc in lancedb_metrics_catalog()}
# Every metric kind emitted by the object store must be described so it is
# surfaced by the bridge (counter, histogram, and gauge).
assert catalog["lance_object_store_requests_total"].kind == "counter"
assert catalog["lance_object_store_request_duration_seconds"].kind == "histogram"
assert catalog["lance_object_store_in_flight_requests"].kind == "gauge"
assert catalog["lance_object_store_retryable_responses_total"].kind == "counter"
# Generate object store activity on the local filesystem (scheme "file").
db = lancedb.connect(str(tmp_path))
table = db.create_table("t", pa.table({"id": pa.array(range(256))}))
assert table.count_rows() == 256
assert table.to_arrow().num_rows == 256
metrics = _metrics_by_name(reader)
requests = metrics["lance_object_store_requests_total"]
points = list(requests.data.data_points)
assert points, "expected at least one request data point"
# Object store metrics are labelled by `operation` and `base` (the store
# scheme, e.g. "file", by default).
assert all("base" in p.attributes and "operation" in p.attributes for p in points)
assert sum(p.value for p in points) > 0
# Histograms are decomposed into bucket / count / sum observable counters.
bucket = metrics["lance_object_store_request_duration_seconds_bucket"]
bucket_points = list(bucket.data.data_points)
assert bucket_points
assert all("le" in p.attributes for p in bucket_points)
# The implicit +Inf bucket must be present and is the cumulative maximum.
assert any(p.attributes["le"] == "+Inf" for p in bucket_points)
count = metrics["lance_object_store_request_duration_seconds_count"]
assert sum(p.value for p in count.data.data_points) > 0
# The `_sum` instrument must also be wired and report positive latency.
duration_sum = metrics["lance_object_store_request_duration_seconds_sum"]
assert sum(p.value for p in duration_sum.data.data_points) > 0
# Unit handling: only `_sum` keeps the histogram's unit (seconds); `_bucket`
# and `_count` observe cumulative counts and are unitless.
assert duration_sum.unit == "s"
assert bucket.unit == ""
assert count.unit == ""
def test_snapshot_empty_before_install_is_safe():
# snapshot is callable regardless of installation state and never raises.
from lancedb._lancedb import snapshot_lancedb_metrics
assert isinstance(snapshot_lancedb_metrics(), list)
def test_instrument_warns_when_recorder_unavailable(monkeypatch):
# A foreign `metrics` recorder already installed -> register returns False;
# instrument_lancedb_metrics must warn and return False without instrumenting.
pytest.importorskip("opentelemetry.sdk.metrics")
import lancedb.otel as otel
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
monkeypatch.setattr(otel, "register_lancedb_metrics_recorder", lambda: False)
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader])
with pytest.warns(UserWarning, match="recorder"):
assert otel.instrument_lancedb_metrics(provider) is False
+18 -63
View File
@@ -780,7 +780,7 @@ name = "cuda-bindings"
version = "13.3.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cuda-pathfinder", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "cuda-pathfinder" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a9/21/8464d133752951c154feafb3b65c297e7d80f301183d220bec4c830f1441/cuda_bindings-13.3.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:120fcc53d57903df529c3486962c56528cba5b7d6c57c99537320ed9922c8b86", size = 6073403, upload-time = "2026-05-29T23:11:36.22Z" },
@@ -815,37 +815,37 @@ wheels = [
[package.optional-dependencies]
cublas = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
cudart = [
{ name = "nvidia-cuda-runtime", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
cufft = [
{ name = "nvidia-cufft", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
cufile = [
{ name = "nvidia-cufile", marker = "sys_platform == 'linux'" },
]
cupti = [
{ name = "nvidia-cuda-cupti", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
curand = [
{ name = "nvidia-curand", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
cusolver = [
{ name = "nvidia-cusolver", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
cusparse = [
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
nvjitlink = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
nvrtc = [
{ name = "nvidia-cuda-nvrtc", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
nvtx = [
{ name = "nvidia-nvtx", marker = "(python_full_version < '3.14' and sys_platform == 'win32') or sys_platform == 'linux'" },
{ name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" },
]
[[package]]
@@ -1909,9 +1909,6 @@ embeddings = [
{ name = "sentencepiece" },
{ name = "torch" },
]
otel = [
{ name = "opentelemetry-api" },
]
pylance = [
{ name = "pylance" },
]
@@ -1926,7 +1923,6 @@ tests = [
{ name = "boto3" },
{ name = "datafusion" },
{ name = "duckdb" },
{ name = "opentelemetry-sdk" },
{ name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
{ name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and python_full_version < '3.14'" },
{ name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.14'" },
@@ -1967,8 +1963,6 @@ requires-dist = [
{ name = "open-clip-torch", marker = "extra == 'clip'" },
{ name = "open-clip-torch", marker = "extra == 'embeddings'", specifier = ">=2.20.0" },
{ name = "openai", marker = "extra == 'embeddings'", specifier = ">=1.6.1" },
{ name = "opentelemetry-api", marker = "extra == 'otel'" },
{ name = "opentelemetry-sdk", marker = "extra == 'tests'", specifier = ">=1.30.0" },
{ name = "overrides", marker = "python_full_version < '3.12'", specifier = ">=0.7" },
{ name = "packaging", specifier = ">=23.0" },
{ name = "pandas", marker = "extra == 'tests'", specifier = ">=1.4" },
@@ -2000,7 +1994,7 @@ requires-dist = [
{ name = "transformers", marker = "extra == 'siglip'", specifier = ">=4.41.0" },
{ name = "typing-extensions", marker = "python_full_version < '3.11' and extra == 'dev'", specifier = ">=4.0.0" },
]
provides-extras = ["azure", "clip", "dev", "docs", "embeddings", "otel", "pylance", "siglip", "tests"]
provides-extras = ["azure", "clip", "dev", "docs", "embeddings", "pylance", "siglip", "tests"]
[[package]]
name = "lomond"
@@ -2781,7 +2775,7 @@ name = "nvidia-cudnn-cu13"
version = "9.19.0.56"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cublas" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" },
@@ -2793,7 +2787,7 @@ name = "nvidia-cufft"
version = "12.0.0.61"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" },
@@ -2823,9 +2817,9 @@ name = "nvidia-cusolver"
version = "12.0.4.66"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cusparse", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-cublas" },
{ name = "nvidia-cusparse" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" },
@@ -2837,7 +2831,7 @@ name = "nvidia-cusparse"
version = "12.6.3.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-nvjitlink", marker = "(python_full_version < '3.14' and sys_platform == 'emscripten') or (python_full_version < '3.14' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" },
{ name = "nvidia-nvjitlink" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" },
@@ -2940,45 +2934,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f6/46/180e14be801a75bc13f234cb1b594b232adeb9c84e60a9ab1832e8333591/openai-2.40.0-py3-none-any.whl", hash = "sha256:2b205637ff214477f9ce9ab035e9f494db0e3fa8f1e599008953735fbf6ff1ff", size = 1350935, upload-time = "2026-06-01T21:48:21.462Z" },
]
[[package]]
name = "opentelemetry-api"
version = "1.43.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ae/cc/e4c9584181f86494df0f6bdec1a4f3280c50db44704dc2a407e994fc87bb/opentelemetry_api-1.43.0.tar.gz", hash = "sha256:107d0d03857ea8fc7c5fcbbbd83f800c281f0d560553d61c1d675fccfd1761c1", size = 73476, upload-time = "2026-06-24T15:19:55.323Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" },
]
[[package]]
name = "opentelemetry-sdk"
version = "1.43.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "opentelemetry-semantic-conventions" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/3e/eb/5041074274ac0956b03637cc039d434569112468e875eddfcc9a0674ce06/opentelemetry_sdk-1.43.0.tar.gz", hash = "sha256:d8187c81c162df9913e4003dd6485f7390d9a24fc17026ec7387b8b8218b08e9", size = 254744, upload-time = "2026-06-24T15:20:08.467Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/49/e3/b17be23af124201c9f52eececd4cc8ddfed1597d37b4ee771895d325805c/opentelemetry_sdk-1.43.0-py3-none-any.whl", hash = "sha256:d1323a547c1ce69d6a069a17a44b7da82bb8b332051ecb074041f87642c86823", size = 178852, upload-time = "2026-06-24T15:19:52.169Z" },
]
[[package]]
name = "opentelemetry-semantic-conventions"
version = "0.64b0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "opentelemetry-api" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/5a/30/5f26df29509eccd86b99b481ac9ffa39da49ba9577cc69071c552ae30447/opentelemetry_semantic_conventions-0.64b0.tar.gz", hash = "sha256:72f76fb2d1582d9d033dd1fcd84532e961e6ff3d90d24ba6fabc72975a83864c", size = 148340, upload-time = "2026-06-24T15:20:09.267Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" },
]
[[package]]
name = "overrides"
version = "7.7.0"
-17
View File
@@ -49,8 +49,6 @@ lance-encoding = { workspace = true }
lance-arrow = { workspace = true }
lance-namespace = { workspace = true }
lance-namespace-impls = { workspace = true }
metrics = { workspace = true, optional = true }
metrics-util = { workspace = true, optional = true }
moka = { workspace = true }
pin-project = { workspace = true }
tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }
@@ -130,12 +128,6 @@ azure = [
"lance-namespace-impls/dir-azure",
"lance-namespace-impls/credential-vendor-azure",
]
cos = ["lance/tencent", "lance-io/tencent"]
goosefs = [
"lance/goosefs",
"lance-io/goosefs",
"lance-namespace-impls/dir-goosefs",
]
huggingface = [
"lance/huggingface",
"lance-io/huggingface",
@@ -149,15 +141,6 @@ remote = [
"lance-namespace-impls/rest",
"lance-namespace-impls/rest-adapter",
]
# Publish LanceDB's internal metrics (currently object store request counts,
# bytes, latency, errors, and throttles) through the `metrics` crate facade,
# and re-export the `metrics` crate as `lancedb::metrics`. Install any
# `metrics`-compatible recorder to collect them.
metrics = ["dep:metrics", "lance/metrics", "lance-io/metrics"]
# Additional adapter on top of `metrics` that installs a process-global recorder
# and exposes a pull-based snapshot/catalog API (see `lancedb::metrics_otel`)
# for bridging metrics into OpenTelemetry or other pull-based exporters.
metrics-otel = ["metrics", "dep:metrics-util"]
fp16kernels = ["lance-linalg/fp16kernels"]
s3-test = []
bedrock = ["dep:aws-sdk-bedrockruntime"]
+6 -91
View File
@@ -342,19 +342,6 @@ impl ListingDatabase {
))
}
fn storage_base_uri(uri: &str) -> String {
let Ok(mut url) = url::Url::parse(uri) else {
return uri.to_string();
};
url.set_query(None);
let Some((storage_scheme, _commit_scheme)) = url.scheme().split_once('+') else {
return url.to_string();
};
let storage_scheme = storage_scheme.to_string();
let _ = url.set_scheme(&storage_scheme);
url.to_string()
}
async fn prepare_namespace_root(
uri: &str,
storage_options: &HashMap<String, String>,
@@ -533,8 +520,6 @@ impl ListingDatabase {
// will add a trailing '?' to the url
url.set_query(None);
let storage_base_uri = Self::storage_base_uri(url.as_str());
let table_base_uri = if let Some(store) = engine {
static WARN_ONCE: std::sync::Once = std::sync::Once::new();
WARN_ONCE.call_once(|| {
@@ -547,6 +532,8 @@ impl ListingDatabase {
url.to_string()
};
let plain_uri = url.to_string();
let session = request
.session
.clone()
@@ -563,13 +550,13 @@ impl ListingDatabase {
};
let (object_store, base_path) = ObjectStore::from_uri_and_params(
session.store_registry(),
&storage_base_uri,
&plain_uri,
&os_params,
)
.await?;
if object_store.is_local() {
Self::try_create_dir(&storage_base_uri).context(CreateDirSnafu {
path: storage_base_uri.clone(),
Self::try_create_dir(&plain_uri).context(CreateDirSnafu {
path: plain_uri.clone(),
})?;
}
@@ -583,7 +570,7 @@ impl ListingDatabase {
};
let namespace_database = Self::connect_namespace_database(
&storage_base_uri,
&table_base_uri,
options.storage_options.clone(),
request.namespace_client_properties.clone(),
request.read_consistency_interval,
@@ -1322,60 +1309,6 @@ mod tests {
(tempdir, db)
}
#[tokio::test]
async fn test_listing_database_root_ops_do_not_create_manifest() {
let tempdir = tempdir().unwrap();
let uri = tempdir.path().to_str().unwrap();
let request = ConnectRequest {
uri: uri.to_string(),
#[cfg(feature = "remote")]
client_config: Default::default(),
options: Default::default(),
namespace_client_properties: Default::default(),
manifest_enabled: false,
read_consistency_interval: None,
session: None,
};
let db = ListingDatabase::connect_with_options(&request)
.await
.unwrap();
assert!(!tempdir.path().join("__manifest").exists());
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
db.create_table(CreateTableRequest {
name: "root_table".to_string(),
namespace_path: vec![],
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
})
.await
.unwrap();
db.open_table(OpenTableRequest {
name: "root_table".to_string(),
namespace_path: vec![],
index_cache_size: None,
lance_read_params: None,
location: None,
namespace_client: None,
managed_versioning: None,
})
.await
.unwrap();
#[allow(deprecated)]
let table_names = db.table_names(TableNamesRequest::default()).await.unwrap();
assert_eq!(table_names, vec!["root_table".to_string()]);
assert!(!tempdir.path().join("__manifest").exists());
}
#[tokio::test]
async fn test_clone_table_basic() {
let (_tempdir, db) = setup_database().await;
@@ -2350,24 +2283,6 @@ mod tests {
assert_eq!(captured.as_deref(), Some("foo=bar"));
}
#[test]
fn test_storage_base_uri_strips_commit_engine_scheme() {
assert_eq!(
ListingDatabase::storage_base_uri("s3+ddb://bucket/prefix?ddbTableName=commit_table"),
"s3://bucket/prefix"
);
assert_eq!(
ListingDatabase::storage_base_uri("s3://bucket/prefix?foo=bar"),
"s3://bucket/prefix"
);
assert_eq!(
ListingDatabase::storage_base_uri("/tmp/lancedb"),
"/tmp/lancedb"
);
}
/// Regression: connecting via a URL-style URI (which goes through
/// `url::Url::parse` and the `query_pairs_mut()` path) must not
/// append a trailing `?` to per-table URIs when the input URI has
-13
View File
@@ -33,11 +33,6 @@
//! - `remote` - Enable remote client to connect to LanceDB cloud.
//! - `huggingface` - Enable HuggingFace Hub integration for loading datasets from the Hub.
//! - `fp16kernels` - Enable FP16 kernels for faster vector search on CPU.
//! - `metrics` - Publish LanceDB's internal metrics through the
//! [`metrics`](https://docs.rs/metrics) crate facade and re-export that crate.
//! Install any `metrics`-compatible recorder to collect them.
//! - `metrics-otel` - Add a pull-based adapter (the `metrics_otel` module) over
//! the `metrics` facade for bridging metrics into OpenTelemetry or similar.
//!
//! ### Quick Start
//!
@@ -179,8 +174,6 @@ pub mod expr;
pub mod index;
pub mod io;
pub mod ipc;
#[cfg(feature = "metrics-otel")]
pub mod metrics_otel;
#[cfg(feature = "polars")]
mod polars_arrow_convertors;
pub mod query;
@@ -201,12 +194,6 @@ pub use connection::{ConnectNamespaceBuilder, Connection};
pub use error::{Error, Result};
use lance_index::vector::ApproxMode as LanceApproxMode;
use lance_linalg::distance::DistanceType as LanceDistanceType;
/// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
/// the `metrics` feature to publish LanceDB's internal metrics; install any
/// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
/// a built-in pull-based adapter.
#[cfg(feature = "metrics")]
pub use metrics;
pub use table::Table;
#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, Default)]
-594
View File
@@ -1,594 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! A pull-based adapter over the [`metrics`] crate facade.
//!
//! LanceDB (through Lance core) publishes metrics — currently object store
//! request counts, bytes, latency, errors, and throttles — through the global
//! [`metrics`] facade without choosing a backend. This module installs a
//! process-global [`metrics::Recorder`] that aggregates those metrics into
//! lock-free cumulative storage and exposes that state as a snapshot, so callers
//! can feed it into a pull-based exporter such as OpenTelemetry.
//!
//! The language bindings build their OpenTelemetry integrations on top of the
//! three public entry points here: [`register_metrics_recorder`],
//! [`metrics_catalog`], and [`snapshot_metrics`].
//!
//! The recorder is *generic*: it records any metric emitted through the facade,
//! keyed by name and labels. Object store metrics are the first producer, but
//! nothing here is specific to them. New metrics flow through automatically;
//! they only need to be described (via the `metrics` `describe_*!` macros) so
//! callers can discover their name, kind, and unit up front.
//!
//! ## Why pull, not push
//!
//! OpenTelemetry collects on its own schedule and invokes observable-instrument
//! callbacks at collection time. Cumulative counters map directly onto OTel's
//! `ObservableCounter` semantics. So the adapter aggregates in Rust and lets the
//! collection thread pull a [`snapshot`](snapshot_metrics) on demand.
//!
//! ## Histograms
//!
//! OpenTelemetry has no asynchronous histogram instrument, so histograms cannot
//! be pulled as-is. Instead each histogram is aggregated into fixed buckets
//! (Prometheus style) and exposed as cumulative `le` bucket counts plus a count
//! and sum, which the caller can surface as observable counters.
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock};
use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit};
use metrics_util::registry::{Registry, Storage};
/// Bucket boundaries used when a histogram has no registered bounds. Covers a
/// broad latency range so unknown histograms still produce useful buckets.
const DEFAULT_BOUNDS: &[f64] = &[
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0,
];
/// The kind of a metric, mirroring the three `metrics` instrument types.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum MetricKind {
Counter,
Gauge,
Histogram,
}
impl MetricKind {
/// The lowercase name of this kind (`"counter"`, `"gauge"`, `"histogram"`).
pub fn as_str(self) -> &'static str {
match self {
Self::Counter => "counter",
Self::Gauge => "gauge",
Self::Histogram => "histogram",
}
}
}
/// A described metric, used to create one exporter instrument per metric up front.
#[derive(Debug, Clone)]
pub struct MetricDescription {
/// The metric name (e.g. `lance_object_store_requests_total`).
pub name: String,
/// Whether the metric is a counter, gauge, or histogram.
pub kind: MetricKind,
/// The canonical unit label, if the producer described one.
pub unit: Option<String>,
/// Human-readable help text describing the metric.
pub description: String,
}
/// The aggregated value of a metric at snapshot time.
#[derive(Debug, Clone)]
pub enum MetricValue {
/// A counter or gauge value.
Scalar(f64),
/// A histogram, decomposed into cumulative `le` buckets plus count and sum.
Histogram {
/// Cumulative `(le, count)` buckets, ending in the implicit `+Inf` bucket.
buckets: Vec<(String, u64)>,
/// Total number of recorded samples.
count: u64,
/// Sum of all recorded sample values.
sum: f64,
},
}
/// One aggregated metric data point exposed to a caller.
#[derive(Debug, Clone)]
pub struct MetricPoint {
/// The metric name.
pub name: String,
/// Whether the point is a counter, gauge, or histogram.
pub kind: MetricKind,
/// The label set for this point (e.g. `operation`, `base`).
pub attributes: HashMap<String, String>,
/// The aggregated value.
pub value: MetricValue,
}
/// Catalog of described metrics, keyed by metric name.
static CATALOG: LazyLock<Mutex<HashMap<String, CatalogEntry>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
struct CatalogEntry {
kind: MetricKind,
unit: Option<String>,
description: String,
}
/// Per-metric histogram bucket boundaries, keyed by metric name. Producers
/// register their recommended bounds before any metric is recorded.
static HISTOGRAM_BOUNDS: LazyLock<RwLock<HashMap<String, Arc<[f64]>>>> =
LazyLock::new(|| RwLock::new(HashMap::new()));
/// The installed recorder's registry, available once installation succeeds.
static REGISTRY: OnceLock<Arc<Registry<Key, LanceStorage>>> = OnceLock::new();
fn bounds_for(name: &str) -> Arc<[f64]> {
HISTOGRAM_BOUNDS
.read()
.unwrap()
.get(name)
.cloned()
.unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS))
}
/// A histogram that buckets samples at record time into fixed boundaries,
/// keeping a cumulative count and sum. Bucketing eagerly keeps memory bounded
/// (unlike retaining raw samples) and produces Prometheus-style `le` buckets.
struct BucketedHistogram {
/// Sorted, finite upper bounds. A sample `v` falls in the first bucket whose
/// bound is `>= v`; samples above all bounds fall in the implicit `+Inf`
/// bucket stored as the final entry of `counts`.
bounds: Arc<[f64]>,
/// Per-bucket (non-cumulative) counts; length is `bounds.len() + 1`.
counts: Box<[AtomicU64]>,
count: AtomicU64,
/// Running sum of recorded values, stored as `f64` bits (there is no atomic
/// f64, so the bit pattern is held in a `u64`; see [`Self::add_to_sum`]).
sum_bits: AtomicU64,
}
// All atomics here use `Ordering::Relaxed`: each metric counter is independent,
// so no happens-before relationship is needed between them, and a snapshot
// reader tolerates slightly stale values. This matches `metrics_util`'s
// `AtomicStorage`.
impl BucketedHistogram {
fn new(bounds: Arc<[f64]>) -> Self {
let counts = (0..bounds.len() + 1)
.map(|_| AtomicU64::new(0))
.collect::<Vec<_>>()
.into_boxed_slice();
Self {
bounds,
counts,
count: AtomicU64::new(0),
sum_bits: AtomicU64::new(0),
}
}
fn add_to_sum(&self, value: f64) {
// No atomic offers an f64 add, so read the current bit pattern, add in
// float space, and CAS it back, retrying if another thread won the race.
let mut current = self.sum_bits.load(Ordering::Relaxed);
loop {
let updated = (f64::from_bits(current) + value).to_bits();
match self.sum_bits.compare_exchange_weak(
current,
updated,
Ordering::Relaxed,
Ordering::Relaxed,
) {
Ok(_) => break,
Err(actual) => current = actual,
}
}
}
/// Cumulative `le` buckets, total count, and sum at this instant.
fn snapshot(&self) -> MetricValue {
let mut cumulative = 0u64;
let mut buckets = Vec::with_capacity(self.bounds.len() + 1);
for (i, bound) in self.bounds.iter().enumerate() {
cumulative += self.counts[i].load(Ordering::Relaxed);
buckets.push((format!("{}", bound), cumulative));
}
cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed);
buckets.push(("+Inf".to_string(), cumulative));
MetricValue::Histogram {
buckets,
count: self.count.load(Ordering::Relaxed),
sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)),
}
}
}
impl metrics::HistogramFn for BucketedHistogram {
fn record(&self, value: f64) {
let idx = self.bounds.partition_point(|&bound| bound < value);
self.counts[idx].fetch_add(1, Ordering::Relaxed);
self.count.fetch_add(1, Ordering::Relaxed);
self.add_to_sum(value);
}
}
/// Storage backing the registry. Counters and gauges are plain atomics (as in
/// `metrics_util`'s `AtomicStorage`); histograms use [`BucketedHistogram`].
struct LanceStorage;
impl Storage<Key> for LanceStorage {
type Counter = Arc<AtomicU64>;
type Gauge = Arc<AtomicU64>;
type Histogram = Arc<BucketedHistogram>;
fn counter(&self, _key: &Key) -> Self::Counter {
Arc::new(AtomicU64::new(0))
}
fn gauge(&self, _key: &Key) -> Self::Gauge {
// The `metrics` facade writes the f64 bit pattern into this `u64` (the
// snapshot decodes it with `f64::from_bits`), matching `AtomicStorage`.
// `0` decodes to `0.0`, the correct initial value.
Arc::new(AtomicU64::new(0))
}
fn histogram(&self, key: &Key) -> Self::Histogram {
Arc::new(BucketedHistogram::new(bounds_for(key.name())))
}
}
struct LanceRecorder {
registry: Arc<Registry<Key, LanceStorage>>,
}
impl LanceRecorder {
fn describe(
&self,
key: KeyName,
kind: MetricKind,
unit: Option<Unit>,
description: SharedString,
) {
CATALOG.lock().unwrap().insert(
key.as_str().to_string(),
CatalogEntry {
kind,
unit: unit.map(|u| u.as_canonical_label().to_string()),
description: description.into_owned(),
},
);
}
}
impl Recorder for LanceRecorder {
fn describe_counter(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
self.describe(key, MetricKind::Counter, unit, description);
}
fn describe_gauge(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
self.describe(key, MetricKind::Gauge, unit, description);
}
fn describe_histogram(&self, key: KeyName, unit: Option<Unit>, description: SharedString) {
self.describe(key, MetricKind::Histogram, unit, description);
}
fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter {
self.registry
.get_or_create_counter(key, |c| Counter::from_arc(c.clone()))
}
fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge {
self.registry
.get_or_create_gauge(key, |g| Gauge::from_arc(g.clone()))
}
fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram {
self.registry
.get_or_create_histogram(key, |h| Histogram::from_arc(h.clone()))
}
}
/// Register the recommended histogram bounds for every metric-emitting
/// subsystem. New subsystems add their `histogram_bounds()` here.
fn register_bounds() {
let mut bounds = HISTOGRAM_BOUNDS.write().unwrap();
for (name, values) in lance_io::object_store::metrics::histogram_bounds() {
bounds.insert((*name).to_string(), Arc::from(*values));
}
}
/// Describe every metric-emitting subsystem so the catalog is populated. Must
/// run after the recorder is installed. New subsystems add their
/// `describe_metrics()` here.
fn describe_all() {
lance_io::object_store::metrics::describe_metrics();
}
fn labels(key: &Key) -> HashMap<String, String> {
key.labels()
.map(|label| (label.key().to_string(), label.value().to_string()))
.collect()
}
fn collect_points(registry: &Registry<Key, LanceStorage>) -> Vec<MetricPoint> {
let mut points = Vec::new();
for (key, handle) in registry.get_counter_handles() {
points.push(MetricPoint {
name: key.name().to_string(),
kind: MetricKind::Counter,
attributes: labels(&key),
// OpenTelemetry observations are float; counts stay well within the
// f64-exact integer range (2^53), so this cast is lossless in practice.
value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64),
});
}
for (key, handle) in registry.get_gauge_handles() {
points.push(MetricPoint {
name: key.name().to_string(),
kind: MetricKind::Gauge,
attributes: labels(&key),
value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))),
});
}
for (key, handle) in registry.get_histogram_handles() {
points.push(MetricPoint {
name: key.name().to_string(),
kind: MetricKind::Histogram,
attributes: labels(&key),
value: handle.snapshot(),
});
}
points
}
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
///
/// Returns `true` if the recorder is installed (now or previously). Returns
/// `false` if a *different* recorder is already installed — `metrics` allows
/// only one global recorder per process, so LanceDB cannot coexist with another.
pub fn register_metrics_recorder() -> bool {
if REGISTRY.get().is_some() {
return true;
}
let registry = Arc::new(Registry::new(LanceStorage));
let recorder = LanceRecorder {
registry: registry.clone(),
};
// Register bounds *before* installing the recorder. Bounds don't depend on
// the recorder, and once it is installed a concurrent histogram emission
// could otherwise create a handle with the fallback bounds and keep them for
// the process lifetime.
register_bounds();
match metrics::set_global_recorder(recorder) {
Ok(()) => {
let _ = REGISTRY.set(registry);
// Describe metrics only after install so the `describe_*!` macros
// route through this recorder and populate the catalog.
describe_all();
true
}
Err(_) => false,
}
}
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
pub fn metrics_catalog() -> Vec<MetricDescription> {
CATALOG
.lock()
.unwrap()
.iter()
.map(|(name, entry)| MetricDescription {
name: name.clone(),
kind: entry.kind,
unit: entry.unit.clone(),
description: entry.description.clone(),
})
.collect()
}
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
/// is installed. The read is lock-free.
pub fn snapshot_metrics() -> Vec<MetricPoint> {
let Some(registry) = REGISTRY.get() else {
return Vec::new();
};
collect_points(registry)
}
#[cfg(test)]
mod tests {
use super::*;
use metrics::HistogramFn;
fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 {
buckets
.iter()
.find(|(b, _)| b == le)
.map(|(_, c)| *c)
.unwrap_or_else(|| panic!("no bucket with le={le}"))
}
#[test]
fn bucketed_histogram_records_cumulative_buckets() {
let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
hist.record(0.05); // le=0.1
hist.record(0.5); // le=1
hist.record(0.5); // le=1
hist.record(50.0); // +Inf
let MetricValue::Histogram {
buckets,
count,
sum,
} = hist.snapshot()
else {
panic!("expected histogram");
};
// Buckets are cumulative (Prometheus `le` semantics).
assert_eq!(bucket_count(&buckets, "0.1"), 1);
assert_eq!(bucket_count(&buckets, "1"), 3);
assert_eq!(bucket_count(&buckets, "10"), 3);
assert_eq!(bucket_count(&buckets, "+Inf"), 4);
assert_eq!(count, 4);
assert!((sum - 51.05).abs() < 1e-9);
}
#[test]
fn bucketed_histogram_boundary_is_inclusive() {
let hist = BucketedHistogram::new(Arc::from([1.0f64].as_slice()));
hist.record(1.0); // exactly the bound -> le=1, not +Inf
let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
panic!("expected histogram");
};
assert_eq!(bucket_count(&buckets, "1"), 1);
assert_eq!(bucket_count(&buckets, "+Inf"), 1);
}
#[test]
fn bucketed_histogram_boundary_is_inclusive_mid_range() {
// A value equal to a middle bound lands in that bucket, not the next.
let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice()));
hist.record(1.0);
let MetricValue::Histogram { buckets, .. } = hist.snapshot() else {
panic!("expected histogram");
};
assert_eq!(bucket_count(&buckets, "0.1"), 0);
assert_eq!(bucket_count(&buckets, "1"), 1);
assert_eq!(bucket_count(&buckets, "10"), 1); // cumulative, so still 1
assert_eq!(bucket_count(&buckets, "+Inf"), 1);
}
#[test]
fn recorder_aggregates_counters_with_labels() {
let registry = Arc::new(Registry::new(LanceStorage));
let recorder = LanceRecorder {
registry: registry.clone(),
};
metrics::with_local_recorder(&recorder, || {
metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
.increment(2);
metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3")
.increment(3);
// A distinct label set must produce a separate point, not merge.
metrics::counter!("test_requests_total", "operation" => "put", "scheme" => "gs")
.increment(7);
});
let scalar = |attrs: &[(&str, &str)]| {
let points = collect_points(&registry);
let point = points
.into_iter()
.find(|p| {
p.name == "test_requests_total"
&& attrs
.iter()
.all(|(k, v)| p.attributes.get(*k).map(String::as_str) == Some(*v))
})
.expect("counter recorded for label set");
assert_eq!(point.kind, MetricKind::Counter);
match point.value {
MetricValue::Scalar(v) => v,
_ => panic!("expected scalar"),
}
};
// Same labels aggregate; distinct labels stay separate.
assert!((scalar(&[("operation", "get"), ("scheme", "s3")]) - 5.0).abs() < 1e-9);
assert!((scalar(&[("operation", "put"), ("scheme", "gs")]) - 7.0).abs() < 1e-9);
}
#[test]
fn recorder_records_gauges() {
let registry = Arc::new(Registry::new(LanceStorage));
let recorder = LanceRecorder {
registry: registry.clone(),
};
// Gauges store the f64 bit pattern in a u64; the snapshot must decode it.
metrics::with_local_recorder(&recorder, || {
metrics::gauge!("test_gauge", "scheme" => "s3").set(3.5);
});
let points = collect_points(&registry);
let point = points
.iter()
.find(|p| p.name == "test_gauge")
.expect("gauge recorded");
assert_eq!(point.kind, MetricKind::Gauge);
assert!(matches!(point.value, MetricValue::Scalar(v) if (v - 3.5).abs() < 1e-9));
}
#[test]
fn recorder_falls_back_to_default_bounds() {
// A histogram with no registered bounds uses DEFAULT_BOUNDS.
let name = "test_unregistered_histogram";
assert!(!HISTOGRAM_BOUNDS.read().unwrap().contains_key(name));
let registry = Arc::new(Registry::new(LanceStorage));
let recorder = LanceRecorder {
registry: registry.clone(),
};
metrics::with_local_recorder(&recorder, || {
metrics::histogram!(name).record(0.02);
});
let points = collect_points(&registry);
let point = points.iter().find(|p| p.name == name).expect("recorded");
let MetricValue::Histogram { buckets, count, .. } = &point.value else {
panic!("expected histogram");
};
assert_eq!(*count, 1);
// DEFAULT_BOUNDS yields one bucket per bound plus the implicit `+Inf`.
assert_eq!(buckets.len(), DEFAULT_BOUNDS.len() + 1);
// 0.02 falls in the le=0.025 bucket (the third DEFAULT_BOUNDS entry).
assert_eq!(bucket_count(buckets, "0.025"), 1);
assert_eq!(bucket_count(buckets, "0.01"), 0);
assert_eq!(bucket_count(buckets, "+Inf"), 1);
}
#[test]
fn recorder_uses_registered_histogram_bounds() {
let name = "test_recorder_bounds_seconds";
HISTOGRAM_BOUNDS
.write()
.unwrap()
.insert(name.to_string(), Arc::from([0.1f64, 1.0].as_slice()));
let registry = Arc::new(Registry::new(LanceStorage));
let recorder = LanceRecorder {
registry: registry.clone(),
};
metrics::with_local_recorder(&recorder, || {
metrics::histogram!(name).record(0.05);
metrics::histogram!(name).record(5.0);
});
let points = collect_points(&registry);
let point = points.iter().find(|p| p.name == name).expect("recorded");
let MetricValue::Histogram { buckets, count, .. } = &point.value else {
panic!("expected histogram");
};
assert_eq!(*count, 2);
assert_eq!(bucket_count(buckets, "0.1"), 1);
assert_eq!(bucket_count(buckets, "+Inf"), 2);
}
#[test]
fn describe_populates_catalog() {
let name = "test_describe_catalog_total";
let registry = Arc::new(Registry::new(LanceStorage));
let recorder = LanceRecorder { registry };
metrics::with_local_recorder(&recorder, || {
metrics::describe_counter!(name, Unit::Count, "a test counter");
});
let catalog = CATALOG.lock().unwrap();
let entry = catalog.get(name).expect("described");
assert_eq!(entry.kind, MetricKind::Counter);
assert_eq!(entry.description, "a test counter");
}
}
+2 -133
View File
@@ -20,7 +20,6 @@ use crate::table::AddColumnsResult;
use crate::table::AddResult;
use crate::table::DeleteResult;
use crate::table::DropColumnsResult;
use crate::table::LsmWriteSpec;
use crate::table::MergeResult;
use crate::table::Tags;
use crate::table::UpdateResult;
@@ -2270,7 +2269,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})
}
async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> Result<()> {
async fn set_lsm_write_spec(&self, spec: crate::table::LsmWriteSpec) -> Result<()> {
use crate::table::LsmWriteSpec;
self.check_mutable().await?;
// Map the spec onto the server's request DTO. `sharding` is internally
@@ -2322,69 +2322,6 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
Ok(())
}
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
// Read counterpart to set/unset, resolved server-side against HEAD. The
// server reads the spec from the `__lance_mem_wal` system index (shard
// column mapped from its Lance field id against the current schema) and
// re-encodes it into the same sophon-owned shape the set endpoint
// accepts — no lance/lancedb types cross the wire. `lsm_write_spec` is
// null when the LSM write path is not enabled for the table.
let request = self.post_read(&format!(
"/v1/table/{}/get_lsm_write_spec/",
self.identifier
));
let (request_id, response) = self.send(request, true).await?;
let response = self.check_table_response(&request_id, response).await?;
let body = response.text().await.err_to_http(request_id.clone())?;
// Mirror of sophon's `Sharding` (internally tagged on `mode`) and
// `LsmWriteSpecBody` / `GetLsmWriteSpecResponse`.
#[derive(Deserialize)]
#[serde(tag = "mode", rename_all = "snake_case")]
enum Sharding {
Unsharded,
Bucket { column: String, num_buckets: u32 },
Identity { column: String },
}
#[derive(Deserialize)]
struct LsmWriteSpecBody {
sharding: Sharding,
#[serde(default)]
maintained_indexes: Vec<String>,
#[serde(default)]
writer_config_defaults: std::collections::HashMap<String, String>,
}
#[derive(Deserialize)]
struct GetLsmWriteSpecResponse {
lsm_write_spec: Option<LsmWriteSpecBody>,
}
let parsed: GetLsmWriteSpecResponse =
serde_json::from_str(&body).map_err(|e| Error::Http {
source: format!("Failed to parse get_lsm_write_spec response: {}", e).into(),
request_id,
status_code: None,
})?;
let Some(body) = parsed.lsm_write_spec else {
// The LSM write path is not enabled for this table.
return Ok(None);
};
let spec = match body.sharding {
Sharding::Bucket {
column,
num_buckets,
} => LsmWriteSpec::bucket(column, num_buckets),
Sharding::Identity { column } => LsmWriteSpec::identity(column),
Sharding::Unsharded => LsmWriteSpec::unsharded(),
}
.with_maintained_indexes(body.maintained_indexes)
.with_writer_config_defaults(body.writer_config_defaults);
Ok(Some(spec))
}
async fn tags(&self) -> Result<Box<dyn Tags + '_>> {
Ok(Box::new(RemoteTags { inner: self }))
}
@@ -5424,74 +5361,6 @@ mod tests {
table.unset_lsm_write_spec().await.unwrap();
}
#[tokio::test]
async fn test_get_lsm_write_spec() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(
request.url().path(),
"/v1/table/my_table/get_lsm_write_spec/"
);
// The server resolves the spec and re-encodes it into the same
// sophon-owned shape the set endpoint accepts (`Sharding` internally
// tagged on `mode`, wrapped in `lsm_write_spec`).
let response = serde_json::json!({
"lsm_write_spec": {
"sharding": { "mode": "bucket", "column": "id", "num_buckets": 4 },
"maintained_indexes": ["id_idx"],
"writer_config_defaults": { "durable_write": "false" },
}
});
http::Response::builder()
.status(200)
.body(response.to_string())
.unwrap()
});
let spec = table
.get_lsm_write_spec()
.await
.unwrap()
.expect("a spec should be reported");
match spec {
crate::table::LsmWriteSpec::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => {
assert_eq!(column, "id");
assert_eq!(num_buckets, 4);
assert_eq!(maintained_indexes, vec!["id_idx".to_string()]);
assert_eq!(
writer_config_defaults
.get("durable_write")
.map(String::as_str),
Some("false")
);
}
other => panic!("expected a bucket spec, got {:?}", other),
}
}
#[tokio::test]
async fn test_get_lsm_write_spec_absent() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(
request.url().path(),
"/v1/table/my_table/get_lsm_write_spec/"
);
// Null spec → the LSM write path is not enabled.
let response = serde_json::json!({ "lsm_write_spec": null });
http::Response::builder()
.status(200)
.body(response.to_string())
.unwrap()
});
assert!(table.get_lsm_write_spec().await.unwrap().is_none());
}
#[tokio::test]
async fn test_wait_for_index() {
let table = _make_table_with_indices(0);
-98
View File
@@ -581,16 +581,6 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
message: "unset_lsm_write_spec is not supported on this table type".into(),
})
}
/// Read the [`LsmWriteSpec`] currently installed on this table, returning
/// `None` when the MemWAL LSM write path is not enabled.
///
/// The default implementation returns `NotSupported`. Implementations that
/// support the MemWAL LSM write path must override this.
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
Err(Error::NotSupported {
message: "get_lsm_write_spec is not supported on this table type".into(),
})
}
/// Drain and close any cached MemWAL shard writers for this table.
///
/// The default implementation is a no-op; table types that maintain
@@ -1548,29 +1538,6 @@ impl Table {
self.inner.unset_lsm_write_spec().await
}
/// Read the [`LsmWriteSpec`] currently installed on this table.
///
/// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no
/// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]).
/// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and
/// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to
/// [`Table::set_lsm_write_spec`].
///
/// # Example
///
/// ```
/// # use lancedb::table::Table;
/// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// if let Some(spec) = table.get_lsm_write_spec().await? {
/// println!("LSM write path enabled: {:?}", spec);
/// }
/// # Ok(())
/// # }
/// ```
pub async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
self.inner.get_lsm_write_spec().await
}
/// Drain and close any cached MemWAL shard writers held for this table.
///
/// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard
@@ -2883,10 +2850,6 @@ impl BaseTable for NativeTable {
merge::lsm::unset_lsm_write_spec(self).await
}
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
merge::lsm::get_lsm_write_spec(self).await
}
async fn close_lsm_writers(&self) -> Result<()> {
merge::lsm::close_lsm_writers(self).await
}
@@ -4448,67 +4411,6 @@ mod tests {
}
}
#[tokio::test]
async fn test_get_lsm_write_spec() {
let tmp_dir = tempdir().unwrap();
let uri = tmp_dir.path().to_str().unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("region", DataType::Utf8, true),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
],
)
.unwrap();
let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
let conn = ConnectBuilder::new(uri)
.read_consistency_interval(Duration::from_secs(0))
.execute()
.await
.unwrap();
let table = conn.create_table("t", reader).execute().await.unwrap();
// No spec installed yet.
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// A real scalar index is needed to name it as a maintained index.
table
.create_index(&["id"], Index::Auto)
.execute()
.await
.unwrap();
let idx_name = table.list_indices().await.unwrap()[0].name.clone();
// Bucket spec round-trips exactly, including the routing column (recovered
// from its field id), maintained indexes, and writer config defaults.
let spec = LsmWriteSpec::bucket("id", 4)
.with_maintained_indexes([idx_name])
.with_writer_config_defaults([("durable_write", "false")]);
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
// After unset, no spec is reported.
table.unset_lsm_write_spec().await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Identity sharding round-trips (column recovered from the schema).
let spec = LsmWriteSpec::identity("region");
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
table.unset_lsm_write_spec().await.unwrap();
// Unsharded round-trips (no routing column).
let spec = LsmWriteSpec::unsharded();
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
}
#[tokio::test]
pub async fn test_stats() {
let tmp_dir = tempdir().unwrap();
+1 -92
View File
@@ -32,7 +32,7 @@ use lance::dataset::mem_wal::{
};
use lance::index::DatasetIndexExt;
use lance_core::datatypes::Schema as LanceSchema;
use lance_index::mem_wal::{MemWalIndexDetails, ShardingField, ShardingSpec};
use lance_index::mem_wal::{MemWalIndexDetails, ShardingSpec};
use tokio::sync::RwLock;
use uuid::Uuid;
@@ -148,97 +148,6 @@ pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> {
Ok(())
}
// =============================================================================
// get_lsm_write_spec
// =============================================================================
/// Read the [`LsmWriteSpec`] currently installed on the table, if any.
///
/// Reconstructs the spec from the MemWAL index's stored details — the same
/// metadata [`set_lsm_write_spec`] writes — read directly via
/// [`mem_wal_index_details`](DatasetMemWalExt::mem_wal_index_details) (the raw
/// manifest record, not the `describe_indices` enrichment path, so it is
/// unaffected by system-index filtering there). Returns `Ok(None)` when no
/// MemWAL index is installed.
#[allow(clippy::redundant_pub_crate)]
pub(crate) async fn get_lsm_write_spec(table: &NativeTable) -> Result<Option<LsmWriteSpec>> {
let dataset = table.dataset.get().await?;
let Some(details) = dataset.mem_wal_index_details().await? else {
return Ok(None);
};
let spec = lsm_write_spec_from_details(&details, dataset.schema())?;
Ok(Some(spec))
}
/// Reconstruct the public [`LsmWriteSpec`] from stored MemWAL index details.
///
/// The sharding transform and its parameters recover the mode; the routing
/// column is resolved from the sharding field's source id back to its schema
/// name (only bucket / identity carry a column). `maintained_indexes` and
/// `writer_config_defaults` are copied verbatim, so the result round-trips
/// what `set_lsm_write_spec` installed.
fn lsm_write_spec_from_details(
details: &MemWalIndexDetails,
schema: &LanceSchema,
) -> Result<LsmWriteSpec> {
let spec = details
.sharding_specs
.first()
.ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: MemWAL index has no sharding spec".to_string(),
})?;
let field = spec.fields.first().ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: MemWAL index has an empty sharding spec".to_string(),
})?;
let base = match field.transform.as_deref() {
Some(BUCKET_TRANSFORM) => {
let num_buckets = field
.parameters
.get(NUM_BUCKETS_PARAM)
.and_then(|s| s.parse::<u32>().ok())
.filter(|n| *n > 0)
.ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(),
})?;
LsmWriteSpec::bucket(sharding_column(field, schema)?, num_buckets)
}
Some(IDENTITY_TRANSFORM) => LsmWriteSpec::identity(sharding_column(field, schema)?),
Some(UNSHARDED_TRANSFORM) => LsmWriteSpec::unsharded(),
other => {
return Err(Error::Runtime {
message: format!(
"get_lsm_write_spec: MemWAL index has an unsupported sharding transform {:?}",
other
),
});
}
};
Ok(base
.with_maintained_indexes(details.maintained_indexes.clone())
.with_writer_config_defaults(details.writer_config_defaults.clone()))
}
/// Resolve the single routing column name from a sharding field's source id.
///
/// `set_lsm_write_spec` records the shard column by its Lance field id, so the
/// name is recovered by looking that id up in the current schema.
fn sharding_column(field: &ShardingField, schema: &LanceSchema) -> Result<String> {
let source_id = *field.source_ids.first().ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: sharding field has no source column".to_string(),
})?;
schema
.field_by_id(source_id)
.map(|f| f.name.clone())
.ok_or_else(|| Error::Runtime {
message: format!(
"get_lsm_write_spec: sharding source field id {} not found in schema",
source_id
),
})
}
// =============================================================================
// close_lsm_writers
// =============================================================================