diff --git a/docs/src/js/interfaces/TableStatistics.md b/docs/src/js/interfaces/TableStatistics.md index e2e8ef34d..4061a9f15 100644 --- a/docs/src/js/interfaces/TableStatistics.md +++ b/docs/src/js/interfaces/TableStatistics.md @@ -18,6 +18,23 @@ Statistics on table fragments *** +### numDeletedRows? + +```ts +optional numDeletedRows: number; +``` + +The number of rows marked as deleted across all fragments of the table + +These rows are not included in `numRows`, but still occupy space on disk +until the table is compacted, so a large value here indicates that the +table should be optimized. Fragments in which every row was deleted are +dropped outright, so their rows are not counted here. + +Absent (`undefined`) when the backend does not report deletion counts. + +*** + ### numIndices ```ts diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index ec259b79b..c3e7968aa 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -1449,4 +1449,27 @@ describe("remote connection jobs surface", () => { }, ); }); + + it("leaves numDeletedRows absent when the server omits it", async () => { + // A server that predates num_deleted_rows says nothing about deleted rows + const statsReply = + '{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":' + + '{"num_fragments":1,"num_small_fragments":0,' + + '"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}'; + + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + const body = path.endsWith("/describe/") + ? JSON.stringify({ name: "t", version: 1, schema: { fields: [] } }) + : statsReply; + res.writeHead(200, { "Content-Type": "application/json" }).end(body); + }, + async (db) => { + const stats = await (await db.openTable("t")).stats(); + expect("numDeletedRows" in stats).toBe(false); + expect(stats.numDeletedRows).toBeUndefined(); + }, + ); + }); }); diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 24cf6e0fd..5a27b852b 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -314,6 +314,7 @@ describe.each(arrowVersions)( }, numIndices: 0, numRows: 3, + numDeletedRows: 0, // Full on-disk size of the two data files, footers and metadata included. totalBytes: 550, }); @@ -324,6 +325,15 @@ describe.each(arrowVersions)( const statsWithIndex = await table.stats(); expect(statsWithIndex.numIndices).toBe(1); expect(statsWithIndex.totalBytes).toBeGreaterThan(550); + + // Both rows with id = 1 are deleted, but that empties the second + // fragment, which is dropped outright rather than kept with a deletion + // file, so only the row marked in the surviving fragment is counted. + await table.delete("id = 1"); + const stats = await table.stats(); + expect(stats.numRows).toBe(1); + expect(stats.numDeletedRows).toBe(1); + expect(stats.fragmentStats.numFragments).toBe(1); }); it("should overwrite data if asked", async () => { diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 79b335c37..ddf5be9f4 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1362,6 +1362,16 @@ pub struct TableStatistics { /// The number of rows in the table pub num_rows: i64, + /// The number of rows marked as deleted across all fragments of the table + /// + /// These rows are not included in `numRows`, but still occupy space on disk + /// until the table is compacted, so a large value here indicates that the + /// table should be optimized. Fragments in which every row was deleted are + /// dropped outright, so their rows are not counted here. + /// + /// Absent (`undefined`) when the backend does not report deletion counts. + pub num_deleted_rows: Option, + /// The number of indices in the table pub num_indices: i64, @@ -1410,6 +1420,7 @@ impl From for TableStatistics { Self { total_bytes: v.total_bytes as i64, num_rows: v.num_rows as i64, + num_deleted_rows: v.num_deleted_rows.map(|n| n as i64), num_indices: v.num_indices as i64, fragment_stats: FragmentStatistics { num_fragments: v.fragment_stats.num_fragments as i64, diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 8824834c2..2275b0a93 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -7329,6 +7329,13 @@ class TableStatistics: and manifests. num_rows: int The total number of rows in the table. + num_deleted_rows: Optional[int] + The total number of rows marked as deleted across all fragments of the + table. These rows are not counted in ``num_rows``, but still occupy space + on disk until the table is compacted, so a large value here indicates + that the table should be optimized. Fragments in which every row was + deleted are dropped outright, so their rows are not counted here. + ``None`` when the backend does not report deletion counts. num_indices: int The total number of indices in the table. fragment_stats: FragmentStatistics @@ -7337,6 +7344,7 @@ class TableStatistics: total_bytes: int num_rows: int + num_deleted_rows: Optional[int] num_indices: int fragment_stats: FragmentStatistics diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 787a39e1d..effe6f1a7 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1474,7 +1474,9 @@ def test_stats(): table = db.create_table("test", [{"id": 1}]) res = table.stats() print(f"{res=}") - assert res == stats + # The server does not report num_deleted_rows, and that unsupported + # state is preserved as None. + assert res == {**stats, "num_deleted_rows": None} @contextlib.contextmanager diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 64f4f4ce9..c579f3037 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4541,22 +4541,25 @@ def test_stats(mem_db: DBConnection): "my_table", data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}], ) - assert len(table) == 2 + table.add([{"text": "baz", "id": 1}]) + assert len(table) == 3 stats = table.stats() print(f"{stats=}") assert stats == { - # Full on-disk size of the data file, footer and metadata included. - "total_bytes": 637, - "num_rows": 2, + # Full on-disk size of the data files, footer and metadata included: + # 637 bytes for the two-row fragment plus 415 for the one-row fragment. + "total_bytes": 1052, + "num_rows": 3, + "num_deleted_rows": 0, "num_indices": 0, "fragment_stats": { - "num_fragments": 1, - "num_small_fragments": 1, + "num_fragments": 2, + "num_small_fragments": 2, "lengths": { - "min": 2, + "min": 1, "max": 2, - "mean": 2, - "p25": 2, + "mean": 1, + "p25": 1, "p50": 2, "p75": 2, "p99": 2, @@ -4564,12 +4567,19 @@ def test_stats(mem_db: DBConnection): }, } + # Both rows with id = 1 are deleted, but that empties the second fragment, + # which is dropped outright rather than kept with a deletion file, so only + # the row marked in the surviving fragment is counted. + table.delete("id = 1") # Index files count toward total_bytes too (only deletion files and # manifests are excluded). table.create_index("id", config=BTree()) stats_with_index = table.stats() assert stats_with_index["num_indices"] == 1 assert stats_with_index["total_bytes"] > stats["total_bytes"] + assert stats_with_index["num_rows"] == 1 + assert stats_with_index["num_deleted_rows"] == 1 + assert stats_with_index["fragment_stats"]["num_fragments"] == 1 def test_create_table_empty_list_with_schema(mem_db: DBConnection): diff --git a/python/src/table.rs b/python/src/table.rs index bb3d0c3c5..0daa09263 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1266,6 +1266,7 @@ impl Table { let dict = PyDict::new(py); dict.set_item("total_bytes", stats.total_bytes)?; dict.set_item("num_rows", stats.num_rows)?; + dict.set_item("num_deleted_rows", stats.num_deleted_rows)?; dict.set_item("num_indices", stats.num_indices)?; let fragment_stats = PyDict::new(py); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index ca8fe644f..e4653fdca 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -14392,4 +14392,18 @@ mod tests { "no retry should have been attempted" ); } + + #[tokio::test] + async fn test_stats_num_deleted_rows_is_none_when_absent() { + // The stats endpoint does not report num_deleted_rows, so the response + // must still parse and value set to None. + let body = r#"{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":0,"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}"#; + let table = Table::new_with_handler("my_table", move |_| { + http::Response::builder() + .status(200) + .body(body.to_string()) + .unwrap() + }); + assert_eq!(table.stats().await.unwrap().num_deleted_rows, None); + } } diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 5179440df..5bc843f42 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -3956,6 +3956,7 @@ impl BaseTable for NativeTable { let num_rows = self.count_rows(None).await?; let num_indices = self.list_indices().await?.len(); let ds = self.dataset.get().await?; + let num_deleted_rows = Some(ds.count_deleted_rows().await?); // Sizes come from the manifest. Summing per-field `bytes_on_disk` instead // would open every data file to read its column metadata, which costs one // IO per fragment and reports 0 for legacy v1 storage. @@ -4020,6 +4021,7 @@ impl BaseTable for NativeTable { let stats = TableStatistics { total_bytes, num_rows, + num_deleted_rows, num_indices, fragment_stats: frag_stats, }; @@ -4056,6 +4058,20 @@ pub struct TableStatistics { /// The number of rows in the table pub num_rows: usize, + /// The number of rows marked as deleted across all fragments of the table + /// + /// Deleted rows are only marked in deletion files and still occupy space on + /// disk until the table is compacted, so a large value here is a good + /// indication that [`Table::optimize`] should be called. + /// + /// These rows are not included in [`Self::num_rows`]. Fragments in which + /// every row was deleted are dropped outright rather than kept with a + /// deletion file, so their rows are not counted here either. + /// + /// `None` means the backend did not report a deletion count. + #[serde(default)] + pub num_deleted_rows: Option, + /// The number of indices in the table pub num_indices: usize, @@ -5884,6 +5900,7 @@ mod tests { res, TableStatistics { num_rows: 250, + num_deleted_rows: Some(0), num_indices: 0, total_bytes: 8969, fragment_stats: FragmentStatistics { @@ -5907,6 +5924,7 @@ mod tests { res, TableStatistics { num_rows: 0, + num_deleted_rows: Some(0), num_indices: 0, total_bytes: 0, fragment_stats: FragmentStatistics { @@ -6124,4 +6142,60 @@ mod tests { stats.fragment_stats.num_fragments ); } + + #[tokio::test] + pub async fn test_stats_num_deleted_rows() { + let tmp_dir = tempdir().unwrap(); + let uri = tmp_dir.path().to_str().unwrap(); + + let conn = ConnectBuilder::new(uri).execute().await.unwrap(); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let batch = |ids: Vec| { + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(ids))]).unwrap() + }; + + let table = conn + .create_table("test_stats_deletions", batch(vec![0, 1])) + .execute() + .await + .unwrap(); + // A second fragment, holding only rows that the delete below removes. + table.add(batch(vec![1])).execute().await.unwrap(); + table.add(batch(vec![1, 2, 3])).execute().await.unwrap(); + + let res = table.stats().await.unwrap(); + assert_eq!(res.num_deleted_rows, Some(0)); + assert_eq!(res.fragment_stats.num_fragments, 3); + + table.delete("id = 1").await.unwrap(); + + let res = table.stats().await.unwrap(); + // 6 rows are inserted over 3 fragments + // 3 rows with id = 1 are deleted (one in each fragment), and that empties the 2nd + // fragment, which is dropped outright rather than kept with a deletion + // file, so only the row marked in the surviving fragment are counted. + assert_eq!(res.num_deleted_rows, Some(2)); + assert_eq!(res.num_rows, 3); + assert_eq!(res.fragment_stats.num_fragments, 2); + // The marked row is are physically there, so it counts towards the + // 3rd fragment length even though it is excluded from num_rows. + assert_eq!(res.fragment_stats.lengths.max, 3); + + // Compaction materializes the deletions. + table + .optimize(OptimizeAction::Compact { + options: CompactionOptions { + target_rows_per_fragment: 1000, + ..Default::default() + }, + remap_options: None, + }) + .await + .unwrap(); + + let res = table.stats().await.unwrap(); + assert_eq!(res.num_deleted_rows, Some(0)); + assert_eq!(res.num_rows, 3); + } }