mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-14 14:46:53 +00:00
put this in the APIs too: python, ts, rust
This commit is contained in:
@@ -8,6 +8,24 @@
|
||||
|
||||
## Properties
|
||||
|
||||
### columnBytes?
|
||||
|
||||
```ts
|
||||
optional columnBytes: Record<string, number>;
|
||||
```
|
||||
|
||||
The compressed on-disk bytes of each column, keyed by dotted field path
|
||||
("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry
|
||||
covering the field's own bytes plus its whole subtree, so a struct
|
||||
reports its total while its children report the breakdown (the
|
||||
top-level entries alone sum to `totalBytes`). Counts data-file bytes
|
||||
only: blob sidecar payloads and index files are not included, and blob
|
||||
columns therefore report just their descriptor bytes. Undefined when
|
||||
the backend provides no per-column breakdown (e.g. older remote
|
||||
servers).
|
||||
|
||||
***
|
||||
|
||||
### fragmentStats
|
||||
|
||||
```ts
|
||||
|
||||
@@ -30,6 +30,8 @@ is also an [asynchronous API client](#connections-asynchronous).
|
||||
|
||||
::: lancedb.table.Table
|
||||
|
||||
::: lancedb.table.TableStatistics
|
||||
|
||||
::: lancedb.table.FragmentStatistics
|
||||
|
||||
::: lancedb.table.FragmentSummaryStats
|
||||
|
||||
@@ -240,9 +240,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
numIndices: 0,
|
||||
numRows: 3,
|
||||
totalBytes: 44,
|
||||
columnBytes: { id: 44 },
|
||||
});
|
||||
});
|
||||
|
||||
it("should report per-column bytes, including nested fields", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const nested = await conn.createTable("nested_stats", [
|
||||
{ id: 1, meta: { a: 1, b: "one" } },
|
||||
{ id: 2, meta: { a: 2, b: "two" } },
|
||||
]);
|
||||
const stats = await nested.stats();
|
||||
const columnBytes = stats.columnBytes!;
|
||||
|
||||
// Nested fields get their own dotted-path entries.
|
||||
expect(Object.keys(columnBytes).sort()).toEqual([
|
||||
"id",
|
||||
"meta",
|
||||
"meta.a",
|
||||
"meta.b",
|
||||
]);
|
||||
expect(columnBytes["meta.a"]).toBeGreaterThan(0);
|
||||
expect(columnBytes["meta.b"]).toBeGreaterThan(0);
|
||||
// A struct's entry covers its whole subtree.
|
||||
expect(columnBytes["meta"]).toBeGreaterThanOrEqual(
|
||||
columnBytes["meta.a"] + columnBytes["meta.b"],
|
||||
);
|
||||
// Top-level entries alone sum to the table total.
|
||||
expect(columnBytes["id"] + columnBytes["meta"]).toBe(stats.totalBytes);
|
||||
});
|
||||
|
||||
it("should overwrite data if asked", async () => {
|
||||
const addRes = await table.add([{ id: 1 }, { id: 2 }]);
|
||||
expect(addRes).toHaveProperty("version");
|
||||
|
||||
@@ -1017,6 +1017,17 @@ pub struct TableStatistics {
|
||||
|
||||
/// Statistics on table fragments
|
||||
pub fragment_stats: FragmentStatistics,
|
||||
|
||||
/// The compressed on-disk bytes of each column, keyed by dotted field path
|
||||
/// ("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry
|
||||
/// covering the field's own bytes plus its whole subtree, so a struct
|
||||
/// reports its total while its children report the breakdown (the
|
||||
/// top-level entries alone sum to `totalBytes`). Counts data-file bytes
|
||||
/// only: blob sidecar payloads and index files are not included, and blob
|
||||
/// columns therefore report just their descriptor bytes. Undefined when
|
||||
/// the backend provides no per-column breakdown (e.g. older remote
|
||||
/// servers).
|
||||
pub column_bytes: Option<HashMap<String, i64>>,
|
||||
}
|
||||
|
||||
#[napi(object)]
|
||||
@@ -1074,6 +1085,12 @@ impl From<lancedb::table::TableStatistics> for TableStatistics {
|
||||
p99: v.fragment_stats.lengths.p99 as i64,
|
||||
},
|
||||
},
|
||||
column_bytes: v.column_bytes.map(|columns| {
|
||||
columns
|
||||
.into_iter()
|
||||
.map(|(path, bytes)| (path, bytes as i64))
|
||||
.collect()
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6144,12 +6144,22 @@ class TableStatistics:
|
||||
The total number of indices in the table.
|
||||
fragment_stats: FragmentStatistics
|
||||
Statistics about fragments in the table.
|
||||
column_bytes: dict[str, int] | None
|
||||
The compressed on-disk bytes of each column, keyed by dotted field path
|
||||
("meta", "meta.geo", "meta.geo.lat"). Every nesting level gets an entry
|
||||
covering the field's own bytes plus its whole subtree, so a struct
|
||||
reports its total while its children report the breakdown (the
|
||||
top-level entries alone sum to ``total_bytes``). Counts data-file bytes
|
||||
only: blob sidecar payloads and index files are not included, and blob
|
||||
columns therefore report just their descriptor bytes. ``None`` when the
|
||||
backend provides no per-column breakdown (e.g. older remote servers).
|
||||
"""
|
||||
|
||||
total_bytes: int
|
||||
num_rows: int
|
||||
num_indices: int
|
||||
fragment_stats: FragmentStatistics
|
||||
column_bytes: Optional[Dict[str, int]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1009,7 +1009,51 @@ def test_stats():
|
||||
table = db.create_table("test", [{"id": 1}])
|
||||
res = table.stats()
|
||||
print(f"{res=}")
|
||||
assert res == stats
|
||||
# A server that omits the per-column breakdown reports it as None
|
||||
# rather than leaving the key out.
|
||||
assert res == {**stats, "column_bytes": None}
|
||||
|
||||
|
||||
def test_stats_column_bytes():
|
||||
stats = {
|
||||
"total_bytes": 38,
|
||||
"num_rows": 2,
|
||||
"num_indices": 0,
|
||||
"fragment_stats": {
|
||||
"num_fragments": 1,
|
||||
"num_small_fragments": 1,
|
||||
"lengths": {
|
||||
"min": 2,
|
||||
"max": 2,
|
||||
"mean": 2,
|
||||
"p25": 2,
|
||||
"p50": 2,
|
||||
"p75": 2,
|
||||
"p99": 2,
|
||||
},
|
||||
},
|
||||
"column_bytes": {"id": 20, "meta": 18, "meta.a": 18},
|
||||
}
|
||||
|
||||
def handler(request):
|
||||
if request.path == "/v1/table/test/create/?mode=create":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(b"{}")
|
||||
elif request.path == "/v1/table/test/stats/":
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(stats).encode())
|
||||
else:
|
||||
print(request.path)
|
||||
request.send_response(404)
|
||||
request.end_headers()
|
||||
|
||||
with mock_lancedb_connection(handler) as db:
|
||||
table = db.create_table("test", [{"id": 1}])
|
||||
assert table.stats() == stats
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
|
||||
@@ -3410,9 +3410,49 @@ def test_stats(mem_db: DBConnection):
|
||||
"p99": 2,
|
||||
},
|
||||
},
|
||||
# Both columns are counted, and the two together make up total_bytes.
|
||||
"column_bytes": {"text": 34, "id": 26},
|
||||
}
|
||||
|
||||
|
||||
def test_stats_column_bytes_nested(mem_db: DBConnection):
|
||||
schema = pa.schema(
|
||||
[
|
||||
pa.field("id", pa.int32()),
|
||||
pa.field(
|
||||
"meta",
|
||||
pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.utf8())]),
|
||||
),
|
||||
]
|
||||
)
|
||||
table = mem_db.create_table(
|
||||
"nested_stats",
|
||||
data=[{"id": i, "meta": {"a": i, "b": f"value-{i}"}} for i in range(100)],
|
||||
schema=schema,
|
||||
)
|
||||
column_bytes = table.stats()["column_bytes"]
|
||||
|
||||
# Nested fields get their own dotted-path entries.
|
||||
assert set(column_bytes) == {"id", "meta", "meta.a", "meta.b"}
|
||||
assert column_bytes["meta.a"] > 0
|
||||
assert column_bytes["meta.b"] > 0
|
||||
# A struct's entry covers its whole subtree.
|
||||
assert column_bytes["meta"] >= column_bytes["meta.a"] + column_bytes["meta.b"]
|
||||
# Top-level entries alone sum to the table total.
|
||||
assert column_bytes["id"] + column_bytes["meta"] == table.stats()["total_bytes"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stats_column_bytes_async(mem_db_async: AsyncConnection):
|
||||
table = await mem_db_async.create_table(
|
||||
"my_table",
|
||||
data=[{"text": "foo", "id": 0}, {"text": "bar", "id": 1}],
|
||||
)
|
||||
stats = await table.stats()
|
||||
assert set(stats["column_bytes"]) == {"text", "id"}
|
||||
assert sum(stats["column_bytes"].values()) == stats["total_bytes"]
|
||||
|
||||
|
||||
def test_create_table_empty_list_with_schema(mem_db: DBConnection):
|
||||
"""Test creating table with empty list data and schema
|
||||
|
||||
|
||||
@@ -944,6 +944,10 @@ impl Table {
|
||||
fragment_stats.set_item("lengths", fragment_lengths)?;
|
||||
dict.set_item("fragment_stats", fragment_stats)?;
|
||||
|
||||
// `None` when the backend reports no per-column breakdown
|
||||
// (e.g. an older remote server).
|
||||
dict.set_item("column_bytes", stats.column_bytes)?;
|
||||
|
||||
Ok(Some(dict.unbind()))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -9764,4 +9764,53 @@ mod tests {
|
||||
.unwrap();
|
||||
branch.stats().await.unwrap();
|
||||
}
|
||||
|
||||
fn stats_handler(body: &'static str) -> Table {
|
||||
Table::new_with_handler("my_table", move |request| {
|
||||
assert_eq!(request.url().path(), "/v1/table/my_table/stats/");
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(body.to_string())
|
||||
.unwrap()
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_stats_column_bytes() {
|
||||
// The dotted paths are opaque to the client: whatever breakdown the
|
||||
// server sends is passed through verbatim, nesting levels and all.
|
||||
let stats = stats_handler(
|
||||
r#"{"total_bytes":56,"num_rows":2,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":1,"lengths":{"min":2,"max":2,"mean":2,"p25":2,"p50":2,"p75":2,"p99":2}},"column_bytes":{"id":20,"meta":36,"meta.a":16,"meta.b":18}}"#,
|
||||
)
|
||||
.stats()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let column_bytes = stats.column_bytes.unwrap();
|
||||
assert_eq!(column_bytes.len(), 4);
|
||||
assert_eq!(column_bytes["id"], 20);
|
||||
assert_eq!(column_bytes["meta"], 36);
|
||||
assert_eq!(column_bytes["meta.a"], 16);
|
||||
assert_eq!(column_bytes["meta.b"], 18);
|
||||
// Top-level entries alone account for the table total.
|
||||
assert_eq!(
|
||||
column_bytes["id"] + column_bytes["meta"],
|
||||
stats.total_bytes as u64
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_stats_column_bytes_absent() {
|
||||
// An older server omits the field entirely. That must degrade to `None`
|
||||
// rather than failing the whole stats response.
|
||||
let stats = stats_handler(
|
||||
r#"{"total_bytes":1,"num_rows":3,"num_indices":0,"fragment_stats":{"num_fragments":1,"num_small_fragments":0,"lengths":{"min":3,"max":3,"mean":3,"p25":3,"p50":3,"p75":3,"p99":3}}}"#,
|
||||
)
|
||||
.stats()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(stats.column_bytes.is_none());
|
||||
assert_eq!(stats.num_rows, 3);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2096,6 +2096,23 @@ impl Table {
|
||||
}
|
||||
|
||||
/// Retrieve statistics on the table
|
||||
///
|
||||
/// See [`TableStatistics`] for the full breakdown, including
|
||||
/// [`TableStatistics::column_bytes`] for the on-disk size of each column.
|
||||
///
|
||||
/// ```
|
||||
/// # use lancedb::Table;
|
||||
/// # async fn column_sizes(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let stats = table.stats().await?;
|
||||
/// println!("{} bytes total", stats.total_bytes);
|
||||
/// if let Some(column_bytes) = &stats.column_bytes {
|
||||
/// for (path, bytes) in column_bytes {
|
||||
/// println!(" {path}: {bytes} bytes");
|
||||
/// }
|
||||
/// }
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn stats(&self) -> Result<TableStatistics> {
|
||||
self.inner.stats().await
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user