feat: add delete rows to table stats (#3783)

add to table stats the number of rows that has
been marked for deletion, but not yet deleted.
note that when a table fragment is removed due to
deletion of all of its rows, these rows are not counted. since the main
use of this feature is to determine when table compaction is needed, the
fact that these rows are not reported should not impact that decision
(deleted table fragments don't need compaction).

Fixes: https://github.com/lancedb/lancedb/issues/3761

Co-authored-by: Claude Opus 5.0 <noreply@anthropic.com>

nodejs/__test__/remote.test.ts

@@ -1449,4 +1449,27 @@ describe("remote connection jobs surface", () => {
       },
     );
   });
+
+  it("leaves numDeletedRows absent when the server omits it", async () => {
+    // A server that predates num_deleted_rows says nothing about deleted rows
+    const statsReply =
+      '{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":' +
+      '{"num_fragments":1,"num_small_fragments":0,' +
+      '"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}';
+
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        const body = path.endsWith("/describe/")
+          ? JSON.stringify({ name: "t", version: 1, schema: { fields: [] } })
+          : statsReply;
+        res.writeHead(200, { "Content-Type": "application/json" }).end(body);
+      },
+      async (db) => {
+        const stats = await (await db.openTable("t")).stats();
+        expect("numDeletedRows" in stats).toBe(false);
+        expect(stats.numDeletedRows).toBeUndefined();
+      },
+    );
+  });
 });

nodejs/__test__/table.test.ts

@@ -314,6 +314,7 @@ describe.each(arrowVersions)(
         },
         numIndices: 0,
         numRows: 3,
+        numDeletedRows: 0,
         // Full on-disk size of the two data files, footers and metadata included.
         totalBytes: 550,
       });
@@ -324,6 +325,15 @@ describe.each(arrowVersions)(
       const statsWithIndex = await table.stats();
       expect(statsWithIndex.numIndices).toBe(1);
       expect(statsWithIndex.totalBytes).toBeGreaterThan(550);
+
+      // Both rows with id = 1 are deleted, but that empties the second
+      // fragment, which is dropped outright rather than kept with a deletion
+      // file, so only the row marked in the surviving fragment is counted.
+      await table.delete("id = 1");
+      const stats = await table.stats();
+      expect(stats.numRows).toBe(1);
+      expect(stats.numDeletedRows).toBe(1);
+      expect(stats.fragmentStats.numFragments).toBe(1);
     });
 
     it("should overwrite data if asked", async () => {

nodejs/src/table.rs

@@ -1362,6 +1362,16 @@ pub struct TableStatistics {
     /// The number of rows in the table
     pub num_rows: i64,
 
+    /// The number of rows marked as deleted across all fragments of the table
+    ///
+    /// These rows are not included in `numRows`, but still occupy space on disk
+    /// until the table is compacted, so a large value here indicates that the
+    /// table should be optimized. Fragments in which every row was deleted are
+    /// dropped outright, so their rows are not counted here.
+    ///
+    /// Absent (`undefined`) when the backend does not report deletion counts.
+    pub num_deleted_rows: Option<i64>,
+
     /// The number of indices in the table
     pub num_indices: i64,
 
@@ -1410,6 +1420,7 @@ impl From<lancedb::table::TableStatistics> for TableStatistics {
         Self {
             total_bytes: v.total_bytes as i64,
             num_rows: v.num_rows as i64,
+            num_deleted_rows: v.num_deleted_rows.map(|n| n as i64),
             num_indices: v.num_indices as i64,
             fragment_stats: FragmentStatistics {
                 num_fragments: v.fragment_stats.num_fragments as i64,