mirror of
https://github.com/lancedb/lancedb.git
synced 2026-01-08 21:02:58 +00:00
feat: add table stats API (#2363)
* Add a new "table stats" API to expose basic table and fragment statistics with local and remote table implementations ### Questions * This is using `calculate_data_stats` to determine total bytes in the table. This seems like a potentially expensive operation - are there any concerns about performance for large datasets? ### Notes * bytes_on_disk seems to be stored at the column level but there does not seem to be a way to easily calculate total bytes per fragment. This may need to be added in lance before we can support fragment size (bytes) statistics. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Added a method to retrieve comprehensive table statistics, including total rows, index counts, storage size, and detailed fragment size metrics such as minimum, maximum, mean, and percentiles. - Enabled fetching of table statistics from remote sources through asynchronous requests. - Extended table interfaces across Python, Rust, and Node.js to support synchronous and asynchronous retrieval of table statistics. - **Tests** - Introduced tests to verify the accuracy of the new table statistics feature for both populated and empty tables. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -157,6 +157,12 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn stats(&self) -> Result<TableStatistics> {
|
||||
let stats = self.inner_ref()?.stats().await.default_error()?;
|
||||
Ok(stats.into())
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
pub async fn update(
|
||||
&self,
|
||||
@@ -555,6 +561,80 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct TableStatistics {
|
||||
/// The total number of bytes in the table
|
||||
pub total_bytes: i64,
|
||||
|
||||
/// The number of rows in the table
|
||||
pub num_rows: i64,
|
||||
|
||||
/// The number of indices in the table
|
||||
pub num_indices: i64,
|
||||
|
||||
/// Statistics on table fragments
|
||||
pub fragment_stats: FragmentStatistics,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct FragmentStatistics {
|
||||
/// The number of fragments in the table
|
||||
pub num_fragments: i64,
|
||||
|
||||
/// The number of uncompacted fragments in the table
|
||||
pub num_small_fragments: i64,
|
||||
|
||||
/// Statistics on the number of rows in the table fragments
|
||||
pub lengths: FragmentSummaryStats,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct FragmentSummaryStats {
|
||||
/// The number of rows in the fragment with the fewest rows
|
||||
pub min: i64,
|
||||
|
||||
/// The number of rows in the fragment with the most rows
|
||||
pub max: i64,
|
||||
|
||||
/// The mean number of rows in the fragments
|
||||
pub mean: i64,
|
||||
|
||||
/// The 25th percentile of number of rows in the fragments
|
||||
pub p25: i64,
|
||||
|
||||
/// The 50th percentile of number of rows in the fragments
|
||||
pub p50: i64,
|
||||
|
||||
/// The 75th percentile of number of rows in the fragments
|
||||
pub p75: i64,
|
||||
|
||||
/// The 99th percentile of number of rows in the fragments
|
||||
pub p99: i64,
|
||||
}
|
||||
|
||||
impl From<lancedb::table::TableStatistics> for TableStatistics {
|
||||
fn from(v: lancedb::table::TableStatistics) -> Self {
|
||||
Self {
|
||||
total_bytes: v.total_bytes as i64,
|
||||
num_rows: v.num_rows as i64,
|
||||
num_indices: v.num_indices as i64,
|
||||
fragment_stats: FragmentStatistics {
|
||||
num_fragments: v.fragment_stats.num_fragments as i64,
|
||||
num_small_fragments: v.fragment_stats.num_small_fragments as i64,
|
||||
lengths: FragmentSummaryStats {
|
||||
min: v.fragment_stats.lengths.min as i64,
|
||||
max: v.fragment_stats.lengths.max as i64,
|
||||
mean: v.fragment_stats.lengths.mean as i64,
|
||||
p25: v.fragment_stats.lengths.p25 as i64,
|
||||
p50: v.fragment_stats.lengths.p50 as i64,
|
||||
p75: v.fragment_stats.lengths.p75 as i64,
|
||||
p99: v.fragment_stats.lengths.p99 as i64,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
pub struct Version {
|
||||
pub version: i64,
|
||||
|
||||
Reference in New Issue
Block a user