rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add delete rows to table stats (#3783)

Browse Source 
add to table stats the number of rows that has
been marked for deletion, but not yet deleted.
note that when a table fragment is removed due to
deletion of all of its rows, these rows are not counted. since the main
use of this feature is to determine when table compaction is needed, the
fact that these rows are not reported should not impact that decision
(deleted table fragments don't need compaction).

Fixes: https://github.com/lancedb/lancedb/issues/3761

Co-authored-by: Claude Opus 5.0 <noreply@anthropic.com>
This commit is contained in:
Yuval Lifshitz
2026-10-07 08:38:31 -07:00
committed by GitHub
co-authored by Claude Opus 5.0
parent becbda70dd
commit 069b2f7d31
10 changed files with 180 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/tests/test_table.py
+19 -9
View File
@@ -4541,22 +4541,25 @@ def test_stats(mem_db: DBConnection):
"my_table",
data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
)
assert len(table) == 2
table.add([{"text": "baz", "id": 1}])
assert len(table) == 3
stats = table.stats()
print(f"{stats=}")
assert stats == {
# Full on-disk size of the data file, footer and metadata included.
"total_bytes": 637,
"num_rows": 2,
# Full on-disk size of the data files, footer and metadata included:
# 637 bytes for the two-row fragment plus 415 for the one-row fragment.
"total_bytes": 1052,
"num_rows": 3,
"num_deleted_rows": 0,
"num_indices": 0,
"fragment_stats": {
"num_fragments": 1,
"num_small_fragments": 1,
"num_fragments": 2,
"num_small_fragments": 2,
"lengths": {
"min": 2,
"min": 1,
"max": 2,
"mean": 2,
"p25": 2,
"mean": 1,
"p25": 1,
"p50": 2,
"p75": 2,
"p99": 2,
@@ -4564,12 +4567,19 @@ def test_stats(mem_db: DBConnection):
},
}
# Both rows with id = 1 are deleted, but that empties the second fragment,
# which is dropped outright rather than kept with a deletion file, so only
# the row marked in the surviving fragment is counted.
table.delete("id = 1")
# Index files count toward total_bytes too (only deletion files and
# manifests are excluded).
table.create_index("id", config=BTree())
stats_with_index = table.stats()
assert stats_with_index["num_indices"] == 1
assert stats_with_index["total_bytes"] > stats["total_bytes"]
assert stats_with_index["num_rows"] == 1
assert stats_with_index["num_deleted_rows"] == 1
assert stats_with_index["fragment_stats"]["num_fragments"] == 1
def test_create_table_empty_list_with_schema(mem_db: DBConnection):