feat: add delete rows to table stats (#3783)

add to table stats the number of rows that has
been marked for deletion, but not yet deleted.
note that when a table fragment is removed due to
deletion of all of its rows, these rows are not counted. since the main
use of this feature is to determine when table compaction is needed, the
fact that these rows are not reported should not impact that decision
(deleted table fragments don't need compaction).

Fixes: https://github.com/lancedb/lancedb/issues/3761

Co-authored-by: Claude Opus 5.0 <noreply@anthropic.com>

docs/src/js/interfaces/TableStatistics.md

@@ -18,6 +18,23 @@ Statistics on table fragments
 
 ***
 
+### numDeletedRows?
+
+```ts
+optional numDeletedRows: number;
+```
+
+The number of rows marked as deleted across all fragments of the table
+
+These rows are not included in `numRows`, but still occupy space on disk
+until the table is compacted, so a large value here indicates that the
+table should be optimized. Fragments in which every row was deleted are
+dropped outright, so their rows are not counted here.
+
+Absent (`undefined`) when the backend does not report deletion counts.
+
+***
+
 ### numIndices
 
 ```ts

nodejs/__test__/remote.test.ts

@@ -1449,4 +1449,27 @@ describe("remote connection jobs surface", () => {
       },
     );
   });
+
+  it("leaves numDeletedRows absent when the server omits it", async () => {
+    // A server that predates num_deleted_rows says nothing about deleted rows
+    const statsReply =
+      '{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":' +
+      '{"num_fragments":1,"num_small_fragments":0,' +
+      '"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}';
+
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        const body = path.endsWith("/describe/")
+          ? JSON.stringify({ name: "t", version: 1, schema: { fields: [] } })
+          : statsReply;
+        res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+      },
+      async (db) => {
+        const stats = await (await db.openTable("t")).stats();
+        expect("numDeletedRows" in stats).toBe(false);
+        expect(stats.numDeletedRows).toBeUndefined();
+      },
+    );
+  });
 });

nodejs/__test__/table.test.ts

@@ -314,6 +314,7 @@ describe.each(arrowVersions)(
         },
         numIndices: 0,
         numRows: 3,
+        numDeletedRows: 0,
         // Full on-disk size of the two data files, footers and metadata included.
         totalBytes: 550,
       });
@@ -324,6 +325,15 @@ describe.each(arrowVersions)(
       const statsWithIndex = await table.stats();
       expect(statsWithIndex.numIndices).toBe(1);
       expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
+
+      // Both rows with id = 1 are deleted, but that empties the second
+      // fragment, which is dropped outright rather than kept with a deletion
+      // file, so only the row marked in the surviving fragment is counted.
+      await table.delete("id = 1");
+      const stats = await table.stats();
+      expect(stats.numRows).toBe(1);
+      expect(stats.numDeletedRows).toBe(1);
+      expect(stats.fragmentStats.numFragments).toBe(1);
     });
 
     it("should overwrite data if asked", async () => {

nodejs/src/table.rs

@@ -1362,6 +1362,16 @@ pub struct TableStatistics {
     /// The number of rows in the table
     pub num_rows: i64,
 
+    /// The number of rows marked as deleted across all fragments of the table
+    ///
+    /// These rows are not included in `numRows`, but still occupy space on disk
+    /// until the table is compacted, so a large value here indicates that the
+    /// table should be optimized. Fragments in which every row was deleted are
+    /// dropped outright, so their rows are not counted here.
+    ///
+    /// Absent (`undefined`) when the backend does not report deletion counts.
+    pub num_deleted_rows: Option<i64>,
+
     /// The number of indices in the table
     pub num_indices: i64,
 
@@ -1410,6 +1420,7 @@ impl From<lancedb::table::TableStatistics> for TableStatistics {
         Self {
             total_bytes: v.total_bytes as i64,
             num_rows: v.num_rows as i64,
+            num_deleted_rows: v.num_deleted_rows.map(|n| n as i64),
             num_indices: v.num_indices as i64,
             fragment_stats: FragmentStatistics {
                 num_fragments: v.fragment_stats.num_fragments as i64,

python/python/lancedb/table.py

@@ -7329,6 +7329,13 @@ class TableStatistics:
         and manifests.
     num_rows: int
         The total number of rows in the table.
+    num_deleted_rows: Optional[int]
+        The total number of rows marked as deleted across all fragments of the
+        table. These rows are not counted in ``num_rows``, but still occupy space
+        on disk until the table is compacted, so a large value here indicates
+        that the table should be optimized. Fragments in which every row was
+        deleted are dropped outright, so their rows are not counted here.
+        ``None`` when the backend does not report deletion counts.
     num_indices: int
         The total number of indices in the table.
     fragment_stats: FragmentStatistics
@@ -7337,6 +7344,7 @@ class TableStatistics:
 
     total_bytes: int
     num_rows: int
+    num_deleted_rows: Optional[int]
     num_indices: int
     fragment_stats: FragmentStatistics
 

python/python/tests/test_remote_db.py

@@ -1474,7 +1474,9 @@ def test_stats():
         table = db.create_table("test", [{"id": 1}])
         res = table.stats()
         print(f"{res=}")
-        assert res == stats
+        # The server does not report num_deleted_rows, and that unsupported
+        # state is preserved as None.
+        assert res == {**stats, "num_deleted_rows": None}
 
 
 @contextlib.contextmanager

python/python/tests/test_table.py

@@ -4541,22 +4541,25 @@ def test_stats(mem_db: DBConnection):
         "my_table",
         data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
     )
-    assert len(table) == 2
+    table.add([{"text": "baz", "id": 1}])
+    assert len(table) == 3
     stats = table.stats()
     print(f"{stats=}")
     assert stats == {
-        # Full on-disk size of the data file, footer and metadata included.
-        "total_bytes": 637,
-        "num_rows": 2,
+        # Full on-disk size of the data files, footer and metadata included:
+        # 637 bytes for the two-row fragment plus 415 for the one-row fragment.
+        "total_bytes": 1052,
+        "num_rows": 3,
+        "num_deleted_rows": 0,
         "num_indices": 0,
         "fragment_stats": {
-            "num_fragments": 1,
-            "num_small_fragments": 1,
+            "num_fragments": 2,
+            "num_small_fragments": 2,
             "lengths": {
-                "min": 2,
+                "min": 1,
                 "max": 2,
-                "mean": 2,
-                "p25": 2,
+                "mean": 1,
+                "p25": 1,
                 "p50": 2,
                 "p75": 2,
                 "p99": 2,
@@ -4564,12 +4567,19 @@ def test_stats(mem_db: DBConnection):
         },
     }
 
+    # Both rows with id = 1 are deleted, but that empties the second fragment,
+    # which is dropped outright rather than kept with a deletion file, so only
+    # the row marked in the surviving fragment is counted.
+    table.delete("id = 1")
     # Index files count toward total_bytes too (only deletion files and
     # manifests are excluded).
     table.create_index("id", config=BTree())
     stats_with_index = table.stats()
     assert stats_with_index["num_indices"] == 1
     assert stats_with_index["total_bytes"] > stats["total_bytes"]
+    assert stats_with_index["num_rows"] == 1
+    assert stats_with_index["num_deleted_rows"] == 1
+    assert stats_with_index["fragment_stats"]["num_fragments"] == 1
 
 
 def test_create_table_empty_list_with_schema(mem_db: DBConnection):

python/src/table.rs

@@ -1266,6 +1266,7 @@ impl Table {
                 let dict = PyDict::new(py);
                 dict.set_item("total_bytes", stats.total_bytes)?;
                 dict.set_item("num_rows", stats.num_rows)?;
+                dict.set_item("num_deleted_rows", stats.num_deleted_rows)?;
                 dict.set_item("num_indices", stats.num_indices)?;
 
                 let fragment_stats = PyDict::new(py);

rust/lancedb/src/remote/table.rs

@@ -14392,4 +14392,18 @@ mod tests {
             "no retry should have been attempted"
         );
     }
+
+    #[tokio::test]
+    async fn test_stats_num_deleted_rows_is_none_when_absent() {
+        // The stats endpoint does not report num_deleted_rows, so the response
+        // must still parse and value set to None.
+        let body = r#"{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":0,"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}"#;
+        let table = Table::new_with_handler("my_table", move |_| {
+            http::Response::builder()
+                .status(200)
+                .body(body.to_string())
+                .unwrap()
+        });
+        assert_eq!(table.stats().await.unwrap().num_deleted_rows, None);
+    }
 }

rust/lancedb/src/table.rs

@@ -3956,6 +3956,7 @@ impl BaseTable for NativeTable {
         let num_rows = self.count_rows(None).await?;
         let num_indices = self.list_indices().await?.len();
         let ds = self.dataset.get().await?;
+        let num_deleted_rows = Some(ds.count_deleted_rows().await?);
         // Sizes come from the manifest. Summing per-field `bytes_on_disk` instead
         // would open every data file to read its column metadata, which costs one
         // IO per fragment and reports 0 for legacy v1 storage.
@@ -4020,6 +4021,7 @@ impl BaseTable for NativeTable {
         let stats = TableStatistics {
             total_bytes,
             num_rows,
+            num_deleted_rows,
             num_indices,
             fragment_stats: frag_stats,
         };
@@ -4056,6 +4058,20 @@ pub struct TableStatistics {
     /// The number of rows in the table
     pub num_rows: usize,
 
+    /// The number of rows marked as deleted across all fragments of the table
+    ///
+    /// Deleted rows are only marked in deletion files and still occupy space on
+    /// disk until the table is compacted, so a large value here is a good
+    /// indication that [`Table::optimize`] should be called.
+    ///
+    /// These rows are not included in [`Self::num_rows`]. Fragments in which
+    /// every row was deleted are dropped outright rather than kept with a
+    /// deletion file, so their rows are not counted here either.
+    ///
+    /// `None` means the backend did not report a deletion count.
+    #[serde(default)]
+    pub num_deleted_rows: Option<usize>,
+
     /// The number of indices in the table
     pub num_indices: usize,
 
@@ -5884,6 +5900,7 @@ mod tests {
             res,
             TableStatistics {
                 num_rows: 250,
+                num_deleted_rows: Some(0),
                 num_indices: 0,
                 total_bytes: 8969,
                 fragment_stats: FragmentStatistics {
@@ -5907,6 +5924,7 @@ mod tests {
             res,
             TableStatistics {
                 num_rows: 0,
+                num_deleted_rows: Some(0),
                 num_indices: 0,
                 total_bytes: 0,
                 fragment_stats: FragmentStatistics {
@@ -6124,4 +6142,60 @@ mod tests {
             stats.fragment_stats.num_fragments
         );
     }
+
+    #[tokio::test]
+    pub async fn test_stats_num_deleted_rows() {
+        let tmp_dir = tempdir().unwrap();
+        let uri = tmp_dir.path().to_str().unwrap();
+
+        let conn = ConnectBuilder::new(uri).execute().await.unwrap();
+
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
+        let batch = |ids: Vec<i32>| {
+            RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(ids))]).unwrap()
+        };
+
+        let table = conn
+            .create_table("test_stats_deletions", batch(vec![0, 1]))
+            .execute()
+            .await
+            .unwrap();
+        // A second fragment, holding only rows that the delete below removes.
+        table.add(batch(vec![1])).execute().await.unwrap();
+        table.add(batch(vec![1, 2, 3])).execute().await.unwrap();
+
+        let res = table.stats().await.unwrap();
+        assert_eq!(res.num_deleted_rows, Some(0));
+        assert_eq!(res.fragment_stats.num_fragments, 3);
+
+        table.delete("id = 1").await.unwrap();
+
+        let res = table.stats().await.unwrap();
+        // 6 rows are inserted over 3 fragments
+        // 3 rows with id = 1 are deleted (one in each fragment), and that empties the 2nd
+        // fragment, which is dropped outright rather than kept with a deletion
+        // file, so only the row marked in the surviving fragment are counted.
+        assert_eq!(res.num_deleted_rows, Some(2));
+        assert_eq!(res.num_rows, 3);
+        assert_eq!(res.fragment_stats.num_fragments, 2);
+        // The marked row is are physically there, so it counts towards the
+        // 3rd fragment length even though it is excluded from num_rows.
+        assert_eq!(res.fragment_stats.lengths.max, 3);
+
+        // Compaction materializes the deletions.
+        table
+            .optimize(OptimizeAction::Compact {
+                options: CompactionOptions {
+                    target_rows_per_fragment: 1000,
+                    ..Default::default()
+                },
+                remap_options: None,
+            })
+            .await
+            .unwrap();
+
+        let res = table.stats().await.unwrap();
+        assert_eq!(res.num_deleted_rows, Some(0));
+        assert_eq!(res.num_rows, 3);
+    }
 }