mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-16 11:08:24 +00:00
285add40dd
Bridges Lance's internal `metrics`-crate instrumentation (object store request counts, bytes, latency, errors, and throttles) into OpenTelemetry, in both the Python and Node bindings, with a shared adapter in the Rust core. This is the LanceDB counterpart to lance-format/lance#7537. ## Rust core (`rust/lancedb`) Two new, **off-by-default** features: - `metrics` — re-exports the [`metrics`](https://docs.rs/metrics) crate as `lancedb::metrics` and turns on Lance's object-store instrumentation. Install any `metrics`-compatible recorder to collect them. - `metrics-otel` — adds `lancedb::metrics_otel`, a pull-based adapter that installs a process-global recorder aggregating into lock-free cumulative storage and exposes a snapshot/catalog API (`register_metrics_recorder`, `metrics_catalog`, `snapshot_metrics`, `MetricPoint`/`MetricValue`/`MetricKind`/`MetricDescription`). Both bindings build on this. ## Python `lancedb.otel.instrument_lancedb_metrics()` registers each metric as an OpenTelemetry observable instrument on the given (or global) `MeterProvider`. Available via the `otel` extra (`pip install lancedb[otel]`), which pulls in only `opentelemetry-api` — the application supplies and configures the SDK. ## Node `instrumentLanceDbMetrics()` provides the equivalent wiring against `@opentelemetry/api`. This is the only public entry point; the underlying recorder/catalog/snapshot functions stay internal. Because OpenTelemetry has no asynchronous histogram instrument, histograms are exported Prometheus-style as `<name>_bucket` (with an `le` attribute), `<name>_count`, and `<name>_sum`. Only `_sum` carries the histogram's unit; `_bucket` and `_count` observe cumulative counts and are unitless. The adapter is enabled by default in the Python and Node builds, and off by default in the Rust crate. ## Notes - Requires Lance ≥ `v9.0.0-beta.19`, which ships the object-store metrics APIs (upstream lance-format/lance#7537, now merged). `main` is already on beta.19, so this is a single feature commit with no dependency bump. - Tests: 8 Rust unit tests, 3 Python tests, 2 Node tests, all covering the end-to-end object-store-metrics → OpenTelemetry path. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
120 lines
3.9 KiB
Rust
120 lines
3.9 KiB
Rust
// SPDX-License-Identifier: Apache-2.0
|
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
//! Node.js bindings over [`lancedb::metrics_otel`].
|
|
//!
|
|
//! The aggregation, catalog, and histogram bucketing all live in the LanceDB
|
|
//! core crate; this module only converts the core snapshot types into napi
|
|
//! objects and exposes the three entry points to JavaScript, where
|
|
//! `lancedb/otel.ts` bridges them into the user's OpenTelemetry `MeterProvider`.
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use lancedb::metrics_otel::{MetricPoint as CoreMetricPoint, MetricValue};
|
|
use napi_derive::napi;
|
|
|
|
/// One cumulative histogram bucket: all samples with value `<= le`.
|
|
#[napi(object)]
|
|
pub struct MetricBucket {
|
|
/// The inclusive upper bound of the bucket, or `"+Inf"` for the final bucket.
|
|
pub le: String,
|
|
/// Cumulative number of samples less than or equal to `le`.
|
|
pub cumulative_count: f64,
|
|
}
|
|
|
|
/// One aggregated metric data point. For counters and gauges only `value` is
|
|
/// set; for histograms `buckets` (cumulative `le` counts), `count`, and `sum`
|
|
/// are set.
|
|
#[napi(object)]
|
|
pub struct MetricPoint {
|
|
pub name: String,
|
|
pub kind: String,
|
|
pub attributes: HashMap<String, String>,
|
|
pub value: Option<f64>,
|
|
pub buckets: Option<Vec<MetricBucket>>,
|
|
pub count: Option<f64>,
|
|
pub sum: Option<f64>,
|
|
}
|
|
|
|
impl From<CoreMetricPoint> for MetricPoint {
|
|
fn from(point: CoreMetricPoint) -> Self {
|
|
let kind = point.kind.as_str().to_string();
|
|
let (value, buckets, count, sum) = match point.value {
|
|
MetricValue::Scalar(v) => (Some(v), None, None, None),
|
|
MetricValue::Histogram {
|
|
buckets,
|
|
count,
|
|
sum,
|
|
} => (
|
|
None,
|
|
Some(
|
|
buckets
|
|
.into_iter()
|
|
// Counts stay well within the f64-exact integer range
|
|
// (2^53), so this cast is lossless in practice and keeps
|
|
// the values plain JS numbers for OpenTelemetry.
|
|
.map(|(le, cumulative_count)| MetricBucket {
|
|
le,
|
|
cumulative_count: cumulative_count as f64,
|
|
})
|
|
.collect(),
|
|
),
|
|
Some(count as f64),
|
|
Some(sum),
|
|
),
|
|
};
|
|
Self {
|
|
name: point.name,
|
|
kind,
|
|
attributes: point.attributes,
|
|
value,
|
|
buckets,
|
|
count,
|
|
sum,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A described metric, used by the JavaScript layer to create instruments up front.
|
|
#[napi(object)]
|
|
pub struct MetricDescription {
|
|
pub name: String,
|
|
pub kind: String,
|
|
pub unit: Option<String>,
|
|
pub description: String,
|
|
}
|
|
|
|
/// Install the LanceDB metrics recorder as the process-global `metrics` recorder.
|
|
///
|
|
/// Returns `true` if the recorder is installed (now or previously). Returns
|
|
/// `false` if a *different* recorder is already installed — `metrics` allows
|
|
/// only one global recorder per process, so LanceDB cannot coexist with another.
|
|
#[napi]
|
|
pub fn register_lancedb_metrics_recorder() -> bool {
|
|
lancedb::metrics_otel::register_metrics_recorder()
|
|
}
|
|
|
|
/// The catalog of described LanceDB metrics. Empty until the recorder is installed.
|
|
#[napi]
|
|
pub fn lancedb_metrics_catalog() -> Vec<MetricDescription> {
|
|
lancedb::metrics_otel::metrics_catalog()
|
|
.into_iter()
|
|
.map(|desc| MetricDescription {
|
|
name: desc.name,
|
|
kind: desc.kind.as_str().to_string(),
|
|
unit: desc.unit,
|
|
description: desc.description,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// A point-in-time snapshot of every recorded metric. Empty until the recorder
|
|
/// is installed.
|
|
#[napi]
|
|
pub fn snapshot_lancedb_metrics() -> Vec<MetricPoint> {
|
|
lancedb::metrics_otel::snapshot_metrics()
|
|
.into_iter()
|
|
.map(MetricPoint::from)
|
|
.collect()
|
|
}
|