mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-07 04:12:59 +00:00
feat: add table stats API (#2363)
* Add a new "table stats" API to expose basic table and fragment statistics with local and remote table implementations ### Questions * This is using `calculate_data_stats` to determine total bytes in the table. This seems like a potentially expensive operation - are there any concerns about performance for large datasets? ### Notes * bytes_on_disk seems to be stored at the column level but there does not seem to be a way to easily calculate total bytes per fragment. This may need to be added in lance before we can support fragment size (bytes) statistics. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Added a method to retrieve comprehensive table statistics, including total rows, index counts, storage size, and detailed fragment size metrics such as minimum, maximum, mean, and percentiles. - Enabled fetching of table statistics from remote sources through asynchronous requests. - Extended table interfaces across Python, Rust, and Node.js to support synchronous and asynchronous retrieval of table statistics. - **Tests** - Introduced tests to verify the accuracy of the new table statistics feature for both populated and empty tables. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -279,6 +279,40 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let stats = inner.stats().await.infer_error()?;
|
||||
Python::with_gil(|py| {
|
||||
let dict = PyDict::new(py);
|
||||
dict.set_item("total_bytes", stats.total_bytes)?;
|
||||
dict.set_item("num_rows", stats.num_rows)?;
|
||||
dict.set_item("num_indices", stats.num_indices)?;
|
||||
|
||||
let fragment_stats = PyDict::new(py);
|
||||
fragment_stats.set_item("num_fragments", stats.fragment_stats.num_fragments)?;
|
||||
fragment_stats.set_item(
|
||||
"num_small_fragments",
|
||||
stats.fragment_stats.num_small_fragments,
|
||||
)?;
|
||||
|
||||
let fragment_lengths = PyDict::new(py);
|
||||
fragment_lengths.set_item("min", stats.fragment_stats.lengths.min)?;
|
||||
fragment_lengths.set_item("max", stats.fragment_stats.lengths.max)?;
|
||||
fragment_lengths.set_item("mean", stats.fragment_stats.lengths.mean)?;
|
||||
fragment_lengths.set_item("p25", stats.fragment_stats.lengths.p25)?;
|
||||
fragment_lengths.set_item("p50", stats.fragment_stats.lengths.p50)?;
|
||||
fragment_lengths.set_item("p75", stats.fragment_stats.lengths.p75)?;
|
||||
fragment_lengths.set_item("p99", stats.fragment_stats.lengths.p99)?;
|
||||
|
||||
fragment_stats.set_item("lengths", fragment_lengths)?;
|
||||
dict.set_item("fragment_stats", fragment_stats)?;
|
||||
|
||||
Ok(Some(dict.unbind()))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn __repr__(&self) -> String {
|
||||
match &self.inner {
|
||||
None => format!("ClosedTable({})", self.name),
|
||||
|
||||
Reference in New Issue
Block a user