From 069b2f7d31d9dcec911e003eb55dddb819157807 Mon Sep 17 00:00:00 2001 From: Yuval Lifshitz Date: Wed, 7 Oct 2026 18:38:31 +0300 Subject: [PATCH] feat: add delete rows to table stats (#3783) add to table stats the number of rows that has been marked for deletion, but not yet deleted. note that when a table fragment is removed due to deletion of all of its rows, these rows are not counted. since the main use of this feature is to determine when table compaction is needed, the fact that these rows are not reported should not impact that decision (deleted table fragments don't need compaction). Fixes: https://github.com/lancedb/lancedb/issues/3761 Co-authored-by: Claude Opus 5.0 --- docs/src/js/interfaces/TableStatistics.md | 17 ++++++ nodejs/__test__/remote.test.ts | 23 +++++++ nodejs/__test__/table.test.ts | 10 +++ nodejs/src/table.rs | 11 ++++ python/python/lancedb/table.py | 8 +++ python/python/tests/test_remote_db.py | 4 +- python/python/tests/test_table.py | 28 ++++++--- python/src/table.rs | 1 + rust/lancedb/src/remote/table.rs | 14 +++++ rust/lancedb/src/table.rs | 74 +++++++++++++++++++++++ 10 files changed, 180 insertions(+), 10 deletions(-) diff --git a/docs/src/js/interfaces/TableStatistics.md b/docs/src/js/interfaces/TableStatistics.md index e2e8ef34d..4061a9f15 100644 --- a/docs/src/js/interfaces/TableStatistics.md +++ b/docs/src/js/interfaces/TableStatistics.md @@ -18,6 +18,23 @@ Statistics on table fragments *** +### numDeletedRows? + +```ts +optional numDeletedRows: number; +``` + +The number of rows marked as deleted across all fragments of the table + +These rows are not included in `numRows`, but still occupy space on disk +until the table is compacted, so a large value here indicates that the +table should be optimized. Fragments in which every row was deleted are +dropped outright, so their rows are not counted here. + +Absent (`undefined`) when the backend does not report deletion counts. + +*** + ### numIndices ```ts diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index ec259b79b..c3e7968aa 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -1449,4 +1449,27 @@ describe("remote connection jobs surface", () => { }, ); }); + + it("leaves numDeletedRows absent when the server omits it", async () => { + // A server that predates num_deleted_rows says nothing about deleted rows + const statsReply = + '{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":' + + '{"num_fragments":1,"num_small_fragments":0,' + + '"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}'; + + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + const body = path.endsWith("/describe/") + ? JSON.stringify({ name: "t", version: 1, schema: { fields: [] } }) + : statsReply; + res.writeHead(200, { "Content-Type": "application/json" }).end(body); + }, + async (db) => { + const stats = await (await db.openTable("t")).stats(); + expect("numDeletedRows" in stats).toBe(false); + expect(stats.numDeletedRows).toBeUndefined(); + }, + ); + }); }); diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 24cf6e0fd..5a27b852b 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -314,6 +314,7 @@ describe.each(arrowVersions)( }, numIndices: 0, numRows: 3, + numDeletedRows: 0, // Full on-disk size of the two data files, footers and metadata included. totalBytes: 550, }); @@ -324,6 +325,15 @@ describe.each(arrowVersions)( const statsWithIndex = await table.stats(); expect(statsWithIndex.numIndices).toBe(1); expect(statsWithIndex.totalBytes).toBeGreaterThan(550); + + // Both rows with id = 1 are deleted, but that empties the second + // fragment, which is dropped outright rather than kept with a deletion + // file, so only the row marked in the surviving fragment is counted. + await table.delete("id = 1"); + const stats = await table.stats(); + expect(stats.numRows).toBe(1); + expect(stats.numDeletedRows).toBe(1); + expect(stats.fragmentStats.numFragments).toBe(1); }); it("should overwrite data if asked", async () => { diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 79b335c37..ddf5be9f4 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1362,6 +1362,16 @@ pub struct TableStatistics { /// The number of rows in the table pub num_rows: i64, + /// The number of rows marked as deleted across all fragments of the table + /// + /// These rows are not included in `numRows`, but still occupy space on disk + /// until the table is compacted, so a large value here indicates that the + /// table should be optimized. Fragments in which every row was deleted are + /// dropped outright, so their rows are not counted here. + /// + /// Absent (`undefined`) when the backend does not report deletion counts. + pub num_deleted_rows: Option, + /// The number of indices in the table pub num_indices: i64, @@ -1410,6 +1420,7 @@ impl From for TableStatistics { Self { total_bytes: v.total_bytes as i64, num_rows: v.num_rows as i64, + num_deleted_rows: v.num_deleted_rows.map(|n| n as i64), num_indices: v.num_indices as i64, fragment_stats: FragmentStatistics { num_fragments: v.fragment_stats.num_fragments as i64, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 8824834c2..2275b0a93 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -7329,6 +7329,13 @@ class TableStatistics: and manifests. num_rows: int The total number of rows in the table. + num_deleted_rows: Optional[int] + The total number of rows marked as deleted across all fragments of the + table. These rows are not counted in ``num_rows``, but still occupy space + on disk until the table is compacted, so a large value here indicates + that the table should be optimized. Fragments in which every row was + deleted are dropped outright, so their rows are not counted here. + ``None`` when the backend does not report deletion counts. num_indices: int The total number of indices in the table. fragment_stats: FragmentStatistics @@ -7337,6 +7344,7 @@ class TableStatistics: total_bytes: int num_rows: int + num_deleted_rows: Optional[int] num_indices: int fragment_stats: FragmentStatistics diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 787a39e1d..effe6f1a7 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1474,7 +1474,9 @@ def test_stats(): table = db.create_table("test", [{"id": 1}]) res = table.stats() print(f"{res=}") - assert res == stats + # The server does not report num_deleted_rows, and that unsupported + # state is preserved as None. + assert res == {**stats, "num_deleted_rows": None} @contextlib.contextmanager diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 64f4f4ce9..c579f3037 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4541,22 +4541,25 @@ def test_stats(mem_db: DBConnection): "my_table", data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}], ) - assert len(table) == 2 + table.add([{"text": "baz", "id": 1}]) + assert len(table) == 3 stats = table.stats() print(f"{stats=}") assert stats == { - # Full on-disk size of the data file, footer and metadata included. - "total_bytes": 637, - "num_rows": 2, + # Full on-disk size of the data files, footer and metadata included: + # 637 bytes for the two-row fragment plus 415 for the one-row fragment. + "total_bytes": 1052, + "num_rows": 3, + "num_deleted_rows": 0, "num_indices": 0, "fragment_stats": { - "num_fragments": 1, - "num_small_fragments": 1, + "num_fragments": 2, + "num_small_fragments": 2, "lengths": { - "min": 2, + "min": 1, "max": 2, - "mean": 2, - "p25": 2, + "mean": 1, + "p25": 1, "p50": 2, "p75": 2, "p99": 2, @@ -4564,12 +4567,19 @@ def test_stats(mem_db: DBConnection): }, } + # Both rows with id = 1 are deleted, but that empties the second fragment, + # which is dropped outright rather than kept with a deletion file, so only + # the row marked in the surviving fragment is counted. + table.delete("id = 1") # Index files count toward total_bytes too (only deletion files and # manifests are excluded). table.create_index("id", config=BTree()) stats_with_index = table.stats() assert stats_with_index["num_indices"] == 1 assert stats_with_index["total_bytes"] > stats["total_bytes"] + assert stats_with_index["num_rows"] == 1 + assert stats_with_index["num_deleted_rows"] == 1 + assert stats_with_index["fragment_stats"]["num_fragments"] == 1 def test_create_table_empty_list_with_schema(mem_db: DBConnection): diff --git a/python/src/table.rs b/python/src/table.rs index bb3d0c3c5..0daa09263 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1266,6 +1266,7 @@ impl Table { let dict = PyDict::new(py); dict.set_item("total_bytes", stats.total_bytes)?; dict.set_item("num_rows", stats.num_rows)?; + dict.set_item("num_deleted_rows", stats.num_deleted_rows)?; dict.set_item("num_indices", stats.num_indices)?; let fragment_stats = PyDict::new(py); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index ca8fe644f..e4653fdca 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -14392,4 +14392,18 @@ mod tests { "no retry should have been attempted" ); } + + #[tokio::test] + async fn test_stats_num_deleted_rows_is_none_when_absent() { + // The stats endpoint does not report num_deleted_rows, so the response + // must still parse and value set to None. + let body = r#"{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":0,"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}"#; + let table = Table::new_with_handler("my_table", move |_| { + http::Response::builder() + .status(200) + .body(body.to_string()) + .unwrap() + }); + assert_eq!(table.stats().await.unwrap().num_deleted_rows, None); + } } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 5179440df..5bc843f42 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -3956,6 +3956,7 @@ impl BaseTable for NativeTable { let num_rows = self.count_rows(None).await?; let num_indices = self.list_indices().await?.len(); let ds = self.dataset.get().await?; + let num_deleted_rows = Some(ds.count_deleted_rows().await?); // Sizes come from the manifest. Summing per-field `bytes_on_disk` instead // would open every data file to read its column metadata, which costs one // IO per fragment and reports 0 for legacy v1 storage. @@ -4020,6 +4021,7 @@ impl BaseTable for NativeTable { let stats = TableStatistics { total_bytes, num_rows, + num_deleted_rows, num_indices, fragment_stats: frag_stats, }; @@ -4056,6 +4058,20 @@ pub struct TableStatistics { /// The number of rows in the table pub num_rows: usize, + /// The number of rows marked as deleted across all fragments of the table + /// + /// Deleted rows are only marked in deletion files and still occupy space on + /// disk until the table is compacted, so a large value here is a good + /// indication that [`Table::optimize`] should be called. + /// + /// These rows are not included in [`Self::num_rows`]. Fragments in which + /// every row was deleted are dropped outright rather than kept with a + /// deletion file, so their rows are not counted here either. + /// + /// `None` means the backend did not report a deletion count. + #[serde(default)] + pub num_deleted_rows: Option, + /// The number of indices in the table pub num_indices: usize, @@ -5884,6 +5900,7 @@ mod tests { res, TableStatistics { num_rows: 250, + num_deleted_rows: Some(0), num_indices: 0, total_bytes: 8969, fragment_stats: FragmentStatistics { @@ -5907,6 +5924,7 @@ mod tests { res, TableStatistics { num_rows: 0, + num_deleted_rows: Some(0), num_indices: 0, total_bytes: 0, fragment_stats: FragmentStatistics { @@ -6124,4 +6142,60 @@ mod tests { stats.fragment_stats.num_fragments ); } + + #[tokio::test] + pub async fn test_stats_num_deleted_rows() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + + let conn = ConnectBuilder::new(uri).execute().await.unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = |ids: Vec| { + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(ids))]).unwrap() + }; + + let table = conn + .create_table("test_stats_deletions", batch(vec![0, 1])) + .execute() + .await + .unwrap(); + // A second fragment, holding only rows that the delete below removes. + table.add(batch(vec![1])).execute().await.unwrap(); + table.add(batch(vec![1, 2, 3])).execute().await.unwrap(); + + let res = table.stats().await.unwrap(); + assert_eq!(res.num_deleted_rows, Some(0)); + assert_eq!(res.fragment_stats.num_fragments, 3); + + table.delete("id = 1").await.unwrap(); + + let res = table.stats().await.unwrap(); + // 6 rows are inserted over 3 fragments + // 3 rows with id = 1 are deleted (one in each fragment), and that empties the 2nd + // fragment, which is dropped outright rather than kept with a deletion + // file, so only the row marked in the surviving fragment are counted. + assert_eq!(res.num_deleted_rows, Some(2)); + assert_eq!(res.num_rows, 3); + assert_eq!(res.fragment_stats.num_fragments, 2); + // The marked row is are physically there, so it counts towards the + // 3rd fragment length even though it is excluded from num_rows. + assert_eq!(res.fragment_stats.lengths.max, 3); + + // Compaction materializes the deletions. + table + .optimize(OptimizeAction::Compact { + options: CompactionOptions { + target_rows_per_fragment: 1000, + ..Default::default() + }, + remap_options: None, + }) + .await + .unwrap(); + + let res = table.stats().await.unwrap(); + assert_eq!(res.num_deleted_rows, Some(0)); + assert_eq!(res.num_rows, 3); + } }