put this in the APIs too: python, ts, rust

This commit is contained in:
Dan Tasse
2026-07-31 13:43:45 -04:00
parent 13b378814f
commit de7065ae50
10 changed files with 229 additions and 1 deletions
+18
View File
@@ -8,6 +8,24 @@
## Properties
### columnBytes?
```ts
optional columnBytes: Record<string, number>;
```
The compressed on-disk bytes of each column, keyed by dotted field path
("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry
covering the field's own bytes plus its whole subtree, so a struct
reports its total while its children report the breakdown (the
top-level entries alone sum to `totalBytes`). Counts data-file bytes
only: blob sidecar payloads and index files are not included, and blob
columns therefore report just their descriptor bytes. Undefined when
the backend provides no per-column breakdown (e.g. older remote
servers).
***
### fragmentStats
```ts
+2
View File
@@ -30,6 +30,8 @@ is also an [asynchronous API client](#connections-asynchronous).
::: lancedb.table.Table
::: lancedb.table.TableStatistics
::: lancedb.table.FragmentStatistics
::: lancedb.table.FragmentSummaryStats
+27
View File
@@ -240,9 +240,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
numIndices: 0,
numRows: 3,
totalBytes: 44,
columnBytes: { id: 44 },
});
});
it("should report per-column bytes, including nested fields", async () => {
const conn = await connect(tmpDir.name);
const nested = await conn.createTable("nested_stats", [
{ id: 1, meta: { a: 1, b: "one" } },
{ id: 2, meta: { a: 2, b: "two" } },
]);
const stats = await nested.stats();
const columnBytes = stats.columnBytes!;
// Nested fields get their own dotted-path entries.
expect(Object.keys(columnBytes).sort()).toEqual([
"id",
"meta",
"meta.a",
"meta.b",
]);
expect(columnBytes["meta.a"]).toBeGreaterThan(0);
expect(columnBytes["meta.b"]).toBeGreaterThan(0);
// A struct's entry covers its whole subtree.
expect(columnBytes["meta"]).toBeGreaterThanOrEqual(
columnBytes["meta.a"] + columnBytes["meta.b"],
);
// Top-level entries alone sum to the table total.
expect(columnBytes["id"] + columnBytes["meta"]).toBe(stats.totalBytes);
});
it("should overwrite data if asked", async () => {
const addRes = await table.add([{ id: 1 }, { id: 2 }]);
expect(addRes).toHaveProperty("version");
+17
View File
@@ -1017,6 +1017,17 @@ pub struct TableStatistics {
/// Statistics on table fragments
pub fragment_stats: FragmentStatistics,
/// The compressed on-disk bytes of each column, keyed by dotted field path
/// ("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry
/// covering the field's own bytes plus its whole subtree, so a struct
/// reports its total while its children report the breakdown (the
/// top-level entries alone sum to `totalBytes`). Counts data-file bytes
/// only: blob sidecar payloads and index files are not included, and blob
/// columns therefore report just their descriptor bytes. Undefined when
/// the backend provides no per-column breakdown (e.g. older remote
/// servers).
pub column_bytes: Option<HashMap<String, i64>>,
}
#[napi(object)]
@@ -1074,6 +1085,12 @@ impl From<lancedb::table::TableStatistics> for TableStatistics {
p99: v.fragment_stats.lengths.p99 as i64,
},
},
column_bytes: v.column_bytes.map(|columns| {
columns
.into_iter()
.map(|(path, bytes)| (path, bytes as i64))
.collect()
}),
}
}
}
+10
View File
@@ -6144,12 +6144,22 @@ class TableStatistics:
The total number of indices in the table.
fragment_stats: FragmentStatistics
Statistics about fragments in the table.
column_bytes: dict[str, int] | None
The compressed on-disk bytes of each column, keyed by dotted field path
("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry
covering the field's own bytes plus its whole subtree, so a struct
reports its total while its children report the breakdown (the
top-level entries alone sum to ``total_bytes``). Counts data-file bytes
only: blob sidecar payloads and index files are not included, and blob
columns therefore report just their descriptor bytes. ``None`` when the
backend provides no per-column breakdown (e.g. older remote servers).
"""
total_bytes: int
num_rows: int
num_indices: int
fragment_stats: FragmentStatistics
column_bytes: Optional[Dict[str, int]] = None
@dataclass
+45 -1
View File
@@ -1009,7 +1009,51 @@ def test_stats():
table = db.create_table("test", [{"id": 1}])
res = table.stats()
print(f"{res=}")
assert res == stats
# A server that omits the per-column breakdown reports it as None
# rather than leaving the key out.
assert res == {**stats, "column_bytes": None}
def test_stats_column_bytes():
stats = {
"total_bytes": 38,
"num_rows": 2,
"num_indices": 0,
"fragment_stats": {
"num_fragments": 1,
"num_small_fragments": 1,
"lengths": {
"min": 2,
"max": 2,
"mean": 2,
"p25": 2,
"p50": 2,
"p75": 2,
"p99": 2,
},
},
"column_bytes": {"id": 20, "meta": 18, "meta.a": 18},
}
def handler(request):
if request.path == "/v1/table/test/create/?mode=create":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
elif request.path == "/v1/table/test/stats/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(stats).encode())
else:
print(request.path)
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
assert table.stats() == stats
@contextlib.contextmanager
+40
View File
@@ -3410,9 +3410,49 @@ def test_stats(mem_db: DBConnection):
"p99": 2,
},
},
# Both columns are counted, and the two together make up total_bytes.
"column_bytes": {"text": 34, "id": 26},
}
def test_stats_column_bytes_nested(mem_db: DBConnection):
schema = pa.schema(
[
pa.field("id", pa.int32()),
pa.field(
"meta",
pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.utf8())]),
),
]
)
table = mem_db.create_table(
"nested_stats",
data=[{"id": i, "meta": {"a": i, "b": f"value-{i}"}} for i in range(100)],
schema=schema,
)
column_bytes = table.stats()["column_bytes"]
# Nested fields get their own dotted-path entries.
assert set(column_bytes) == {"id", "meta", "meta.a", "meta.b"}
assert column_bytes["meta.a"] > 0
assert column_bytes["meta.b"] > 0
# A struct's entry covers its whole subtree.
assert column_bytes["meta"] >= column_bytes["meta.a"] + column_bytes["meta.b"]
# Top-level entries alone sum to the table total.
assert column_bytes["id"] + column_bytes["meta"] == table.stats()["total_bytes"]
@pytest.mark.asyncio
async def test_stats_column_bytes_async(mem_db_async: AsyncConnection):
table = await mem_db_async.create_table(
"my_table",
data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
)
stats = await table.stats()
assert set(stats["column_bytes"]) == {"text", "id"}
assert sum(stats["column_bytes"].values()) == stats["total_bytes"]
def test_create_table_empty_list_with_schema(mem_db: DBConnection):
"""Test creating table with empty list data and schema
+4
View File
@@ -944,6 +944,10 @@ impl Table {
fragment_stats.set_item("lengths", fragment_lengths)?;
dict.set_item("fragment_stats", fragment_stats)?;
// `None` when the backend reports no per-column breakdown
// (e.g. an older remote server).
dict.set_item("column_bytes", stats.column_bytes)?;
Ok(Some(dict.unbind()))
})
})
+49
View File
@@ -9764,4 +9764,53 @@ mod tests {
.unwrap();
branch.stats().await.unwrap();
}
fn stats_handler(body: &'static str) -> Table {
Table::new_with_handler("my_table", move |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/stats/");
http::Response::builder()
.status(200)
.body(body.to_string())
.unwrap()
})
}
#[tokio::test]
async fn test_stats_column_bytes() {
// The dotted paths are opaque to the client: whatever breakdown the
// server sends is passed through verbatim, nesting levels and all.
let stats = stats_handler(
r#"{"total_bytes":56,"num_rows":2,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":1,"lengths":{"min":2,"max":2,"mean":2,"p25":2,"p50":2,"p75":2,"p99":2}},"column_bytes":{"id":20,"meta":36,"meta.a":16,"meta.b":18}}"#,
)
.stats()
.await
.unwrap();
let column_bytes = stats.column_bytes.unwrap();
assert_eq!(column_bytes.len(), 4);
assert_eq!(column_bytes["id"], 20);
assert_eq!(column_bytes["meta"], 36);
assert_eq!(column_bytes["meta.a"], 16);
assert_eq!(column_bytes["meta.b"], 18);
// Top-level entries alone account for the table total.
assert_eq!(
column_bytes["id"] + column_bytes["meta"],
stats.total_bytes as u64
);
}
#[tokio::test]
async fn test_stats_column_bytes_absent() {
// An older server omits the field entirely. That must degrade to `None`
// rather than failing the whole stats response.
let stats = stats_handler(
r#"{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":0,"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}"#,
)
.stats()
.await
.unwrap();
assert!(stats.column_bytes.is_none());
assert_eq!(stats.num_rows, 3);
}
}
+17
View File
@@ -2096,6 +2096,23 @@ impl Table {
}
/// Retrieve statistics on the table
///
/// See [`TableStatistics`] for the full breakdown, including
/// [`TableStatistics::column_bytes`] for the on-disk size of each column.
///
/// ```
/// # use lancedb::Table;
/// # async fn column_sizes(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// let stats = table.stats().await?;
/// println!("{} bytes total", stats.total_bytes);
/// if let Some(column_bytes) = &stats.column_bytes {
/// for (path, bytes) in column_bytes {
/// println!(" {path}: {bytes} bytes");
/// }
/// }
/// # Ok(())
/// # }
/// ```
pub async fn stats(&self) -> Result<TableStatistics> {
self.inner.stats().await
}