mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-15 10:38:34 +00:00
Will Jones
285add40dd
Bridges Lance's internal `metrics`-crate instrumentation (object store request counts, bytes, latency, errors, and throttles) into OpenTelemetry, in both the Python and Node bindings, with a shared adapter in the Rust core. This is the LanceDB counterpart to lance-format/lance#7537. ## Rust core (`rust/lancedb`) Two new, **off-by-default** features: - `metrics` — re-exports the [`metrics`](https://docs.rs/metrics) crate as `lancedb::metrics` and turns on Lance's object-store instrumentation. Install any `metrics`-compatible recorder to collect them. - `metrics-otel` — adds `lancedb::metrics_otel`, a pull-based adapter that installs a process-global recorder aggregating into lock-free cumulative storage and exposes a snapshot/catalog API (`register_metrics_recorder`, `metrics_catalog`, `snapshot_metrics`, `MetricPoint`/`MetricValue`/`MetricKind`/`MetricDescription`). Both bindings build on this. ## Python `lancedb.otel.instrument_lancedb_metrics()` registers each metric as an OpenTelemetry observable instrument on the given (or global) `MeterProvider`. Available via the `otel` extra (`pip install lancedb[otel]`), which pulls in only `opentelemetry-api` — the application supplies and configures the SDK. ## Node `instrumentLanceDbMetrics()` provides the equivalent wiring against `@opentelemetry/api`. This is the only public entry point; the underlying recorder/catalog/snapshot functions stay internal. Because OpenTelemetry has no asynchronous histogram instrument, histograms are exported Prometheus-style as `<name>_bucket` (with an `le` attribute), `<name>_count`, and `<name>_sum`. Only `_sum` carries the histogram's unit; `_bucket` and `_count` observe cumulative counts and are unitless. The adapter is enabled by default in the Python and Node builds, and off by default in the Rust crate. ## Notes - Requires Lance ≥ `v9.0.0-beta.19`, which ships the object-store metrics APIs (upstream lance-format/lance#7537, now merged). `main` is already on beta.19, so this is a single feature commit with no dependency bump. - Tests: 8 Rust unit tests, 3 Python tests, 2 Node tests, all covering the end-to-end object-store-metrics → OpenTelemetry path. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
84 lines
3.7 KiB
TOML
84 lines
3.7 KiB
TOML
[workspace]
|
|
members = ["rust/lancedb", "nodejs", "python"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
authors = ["LanceDB Devs <dev@lancedb.com>"]
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/lancedb/lancedb"
|
|
description = "Serverless, low-latency vector database for AI applications"
|
|
keywords = ["lancedb", "lance", "database", "vector", "search"]
|
|
categories = ["database-implementations"]
|
|
rust-version = "1.91.0"
|
|
|
|
[workspace.dependencies]
|
|
lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
|
ahash = "0.8"
|
|
# Note that this one does not include pyarrow
|
|
arrow = { version = "58.0.0", optional = false }
|
|
arrow-array = "58.0.0"
|
|
arrow-buffer = "58.0.0"
|
|
arrow-data = "58.0.0"
|
|
arrow-ipc = "58.0.0"
|
|
arrow-ord = "58.0.0"
|
|
arrow-schema = "58.0.0"
|
|
arrow-select = "58.0.0"
|
|
arrow-cast = "58.0.0"
|
|
async-trait = "0"
|
|
datafusion = { version = "53.0.0", default-features = false }
|
|
datafusion-catalog = "53.0.0"
|
|
datafusion-common = { version = "53.0.0", default-features = false }
|
|
datafusion-execution = "53.0.0"
|
|
datafusion-expr = "53.0.0"
|
|
datafusion-functions = "53.0.0"
|
|
datafusion-physical-plan = "53.0.0"
|
|
datafusion-physical-expr = "53.0.0"
|
|
datafusion-sql = "53.0.0"
|
|
env_logger = "0.11"
|
|
half = { "version" = "2.7.1", default-features = false, features = [
|
|
"num-traits",
|
|
] }
|
|
futures = "0"
|
|
log = "0.4"
|
|
metrics = "0.24"
|
|
metrics-util = "0.19"
|
|
moka = { version = "0.12", features = ["future"] }
|
|
object_store = "0.13.2"
|
|
pin-project = "1.0.7"
|
|
rand = "0.9"
|
|
snafu = "0.8"
|
|
url = "2"
|
|
num-traits = "0.2"
|
|
regex = "1.10"
|
|
lazy_static = "1"
|
|
semver = "1.0.25"
|
|
chrono = "0.4"
|
|
|
|
[profile.ci]
|
|
debug = "line-tables-only"
|
|
inherits = "dev"
|
|
incremental = false
|
|
|
|
# This rule applies to every package except workspace members (dependencies
|
|
# such as `arrow` and `tokio`). It disables debug info and related features on
|
|
# dependencies so their binaries stay smaller, improving cache reuse.
|
|
[profile.ci.package."*"]
|
|
debug = false
|
|
debug-assertions = false
|
|
strip = "debuginfo"
|
|
incremental = false
|