From de7065ae5059b91fb4ae12754d6eff333c2acf38 Mon Sep 17 00:00:00 2001 From: Dan Tasse Date: Fri, 31 Jul 2026 13:43:45 -0400 Subject: [PATCH] put this in the APIs too: python, ts, rust --- docs/src/js/interfaces/TableStatistics.md | 18 +++++++++ docs/src/python/python.md | 2 + nodejs/__test__/table.test.ts | 27 +++++++++++++ nodejs/src/table.rs | 17 ++++++++ python/python/lancedb/table.py | 10 +++++ python/python/tests/test_remote_db.py | 46 ++++++++++++++++++++- python/python/tests/test_table.py | 40 ++++++++++++++++++ python/src/table.rs | 4 ++ rust/lancedb/src/remote/table.rs | 49 +++++++++++++++++++++++ rust/lancedb/src/table.rs | 17 ++++++++ 10 files changed, 229 insertions(+), 1 deletion(-) diff --git a/docs/src/js/interfaces/TableStatistics.md b/docs/src/js/interfaces/TableStatistics.md index e19cba119..dcaf5d66f 100644 --- a/docs/src/js/interfaces/TableStatistics.md +++ b/docs/src/js/interfaces/TableStatistics.md @@ -8,6 +8,24 @@ ## Properties +### columnBytes? + +```ts +optional columnBytes: Record; +``` + +The compressed on-disk bytes of each column, keyed by dotted field path +("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry +covering the field's own bytes plus its whole subtree, so a struct +reports its total while its children report the breakdown (the +top-level entries alone sum to `totalBytes`). Counts data-file bytes +only: blob sidecar payloads and index files are not included, and blob +columns therefore report just their descriptor bytes. Undefined when +the backend provides no per-column breakdown (e.g. older remote +servers). + +*** + ### fragmentStats ```ts diff --git a/docs/src/python/python.md b/docs/src/python/python.md index dd60451cc..731045e3d 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -30,6 +30,8 @@ is also an [asynchronous API client](#connections-asynchronous). ::: lancedb.table.Table +::: lancedb.table.TableStatistics + ::: lancedb.table.FragmentStatistics ::: lancedb.table.FragmentSummaryStats diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 41025047e..1725fd13e 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -240,9 +240,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( numIndices: 0, numRows: 3, totalBytes: 44, + columnBytes: { id: 44 }, }); }); + it("should report per-column bytes, including nested fields", async () => { + const conn = await connect(tmpDir.name); + const nested = await conn.createTable("nested_stats", [ + { id: 1, meta: { a: 1, b: "one" } }, + { id: 2, meta: { a: 2, b: "two" } }, + ]); + const stats = await nested.stats(); + const columnBytes = stats.columnBytes!; + + // Nested fields get their own dotted-path entries. + expect(Object.keys(columnBytes).sort()).toEqual([ + "id", + "meta", + "meta.a", + "meta.b", + ]); + expect(columnBytes["meta.a"]).toBeGreaterThan(0); + expect(columnBytes["meta.b"]).toBeGreaterThan(0); + // A struct's entry covers its whole subtree. + expect(columnBytes["meta"]).toBeGreaterThanOrEqual( + columnBytes["meta.a"] + columnBytes["meta.b"], + ); + // Top-level entries alone sum to the table total. + expect(columnBytes["id"] + columnBytes["meta"]).toBe(stats.totalBytes); + }); + it("should overwrite data if asked", async () => { const addRes = await table.add([{ id: 1 }, { id: 2 }]); expect(addRes).toHaveProperty("version"); diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index d1489034f..25cc0ff17 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1017,6 +1017,17 @@ pub struct TableStatistics { /// Statistics on table fragments pub fragment_stats: FragmentStatistics, + + /// The compressed on-disk bytes of each column, keyed by dotted field path + /// ("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry + /// covering the field's own bytes plus its whole subtree, so a struct + /// reports its total while its children report the breakdown (the + /// top-level entries alone sum to `totalBytes`). Counts data-file bytes + /// only: blob sidecar payloads and index files are not included, and blob + /// columns therefore report just their descriptor bytes. Undefined when + /// the backend provides no per-column breakdown (e.g. older remote + /// servers). + pub column_bytes: Option>, } #[napi(object)] @@ -1074,6 +1085,12 @@ impl From for TableStatistics { p99: v.fragment_stats.lengths.p99 as i64, }, }, + column_bytes: v.column_bytes.map(|columns| { + columns + .into_iter() + .map(|(path, bytes)| (path, bytes as i64)) + .collect() + }), } } } diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 26d78a427..7e3f6002c 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6144,12 +6144,22 @@ class TableStatistics: The total number of indices in the table. fragment_stats: FragmentStatistics Statistics about fragments in the table. + column_bytes: dict[str, int] | None + The compressed on-disk bytes of each column, keyed by dotted field path + ("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry + covering the field's own bytes plus its whole subtree, so a struct + reports its total while its children report the breakdown (the + top-level entries alone sum to ``total_bytes``). Counts data-file bytes + only: blob sidecar payloads and index files are not included, and blob + columns therefore report just their descriptor bytes. ``None`` when the + backend provides no per-column breakdown (e.g. older remote servers). """ total_bytes: int num_rows: int num_indices: int fragment_stats: FragmentStatistics + column_bytes: Optional[Dict[str, int]] = None @dataclass diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index dd46628f1..a7b211ea5 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1009,7 +1009,51 @@ def test_stats(): table = db.create_table("test", [{"id": 1}]) res = table.stats() print(f"{res=}") - assert res == stats + # A server that omits the per-column breakdown reports it as None + # rather than leaving the key out. + assert res == {**stats, "column_bytes": None} + + +def test_stats_column_bytes(): + stats = { + "total_bytes": 38, + "num_rows": 2, + "num_indices": 0, + "fragment_stats": { + "num_fragments": 1, + "num_small_fragments": 1, + "lengths": { + "min": 2, + "max": 2, + "mean": 2, + "p25": 2, + "p50": 2, + "p75": 2, + "p99": 2, + }, + }, + "column_bytes": {"id": 20, "meta": 18, "meta.a": 18}, + } + + def handler(request): + if request.path == "/v1/table/test/create/?mode=create": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}") + elif request.path == "/v1/table/test/stats/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(stats).encode()) + else: + print(request.path) + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + table = db.create_table("test", [{"id": 1}]) + assert table.stats() == stats @contextlib.contextmanager diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 2a34084cf..2eb533ac3 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -3410,9 +3410,49 @@ def test_stats(mem_db: DBConnection): "p99": 2, }, }, + # Both columns are counted, and the two together make up total_bytes. + "column_bytes": {"text": 34, "id": 26}, } +def test_stats_column_bytes_nested(mem_db: DBConnection): + schema = pa.schema( + [ + pa.field("id", pa.int32()), + pa.field( + "meta", + pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.utf8())]), + ), + ] + ) + table = mem_db.create_table( + "nested_stats", + data=[{"id": i, "meta": {"a": i, "b": f"value-{i}"}} for i in range(100)], + schema=schema, + ) + column_bytes = table.stats()["column_bytes"] + + # Nested fields get their own dotted-path entries. + assert set(column_bytes) == {"id", "meta", "meta.a", "meta.b"} + assert column_bytes["meta.a"] > 0 + assert column_bytes["meta.b"] > 0 + # A struct's entry covers its whole subtree. + assert column_bytes["meta"] >= column_bytes["meta.a"] + column_bytes["meta.b"] + # Top-level entries alone sum to the table total. + assert column_bytes["id"] + column_bytes["meta"] == table.stats()["total_bytes"] + + +@pytest.mark.asyncio +async def test_stats_column_bytes_async(mem_db_async: AsyncConnection): + table = await mem_db_async.create_table( + "my_table", + data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}], + ) + stats = await table.stats() + assert set(stats["column_bytes"]) == {"text", "id"} + assert sum(stats["column_bytes"].values()) == stats["total_bytes"] + + def test_create_table_empty_list_with_schema(mem_db: DBConnection): """Test creating table with empty list data and schema diff --git a/python/src/table.rs b/python/src/table.rs index 84ba91f39..4bc2850b9 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -944,6 +944,10 @@ impl Table { fragment_stats.set_item("lengths", fragment_lengths)?; dict.set_item("fragment_stats", fragment_stats)?; + // `None` when the backend reports no per-column breakdown + // (e.g. an older remote server). + dict.set_item("column_bytes", stats.column_bytes)?; + Ok(Some(dict.unbind())) }) }) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index b668f6d67..21419ef14 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -9764,4 +9764,53 @@ mod tests { .unwrap(); branch.stats().await.unwrap(); } + + fn stats_handler(body: &'static str) -> Table { + Table::new_with_handler("my_table", move |request| { + assert_eq!(request.url().path(), "/v1/table/my_table/stats/"); + http::Response::builder() + .status(200) + .body(body.to_string()) + .unwrap() + }) + } + + #[tokio::test] + async fn test_stats_column_bytes() { + // The dotted paths are opaque to the client: whatever breakdown the + // server sends is passed through verbatim, nesting levels and all. + let stats = stats_handler( + r#"{"total_bytes":56,"num_rows":2,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":1,"lengths":{"min":2,"max":2,"mean":2,"p25":2,"p50":2,"p75":2,"p99":2}},"column_bytes":{"id":20,"meta":36,"meta.a":16,"meta.b":18}}"#, + ) + .stats() + .await + .unwrap(); + + let column_bytes = stats.column_bytes.unwrap(); + assert_eq!(column_bytes.len(), 4); + assert_eq!(column_bytes["id"], 20); + assert_eq!(column_bytes["meta"], 36); + assert_eq!(column_bytes["meta.a"], 16); + assert_eq!(column_bytes["meta.b"], 18); + // Top-level entries alone account for the table total. + assert_eq!( + column_bytes["id"] + column_bytes["meta"], + stats.total_bytes as u64 + ); + } + + #[tokio::test] + async fn test_stats_column_bytes_absent() { + // An older server omits the field entirely. That must degrade to `None` + // rather than failing the whole stats response. + let stats = stats_handler( + r#"{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":0,"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}"#, + ) + .stats() + .await + .unwrap(); + + assert!(stats.column_bytes.is_none()); + assert_eq!(stats.num_rows, 3); + } } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 30da739fb..91c2c45bc 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -2096,6 +2096,23 @@ impl Table { } /// Retrieve statistics on the table + /// + /// See [`TableStatistics`] for the full breakdown, including + /// [`TableStatistics::column_bytes`] for the on-disk size of each column. + /// + /// ``` + /// # use lancedb::Table; + /// # async fn column_sizes(table: &Table) -> Result<(), Box> { + /// let stats = table.stats().await?; + /// println!("{} bytes total", stats.total_bytes); + /// if let Some(column_bytes) = &stats.column_bytes { + /// for (path, bytes) in column_bytes { + /// println!(" {path}: {bytes} bytes"); + /// } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn stats(&self) -> Result { self.inner.stats().await }