From 8a83aa571bf1868a6d530f48095e4e4c34827029 Mon Sep 17 00:00:00 2001 From: Daniel Rammer Date: Tue, 7 Jul 2026 16:33:46 -0500 Subject: [PATCH] refactor: parse LsmWriteSpec directly from serialized index details The server serializes the resolved LsmWriteSpec (shard column already mapped from its field id) into the MemWAL index's `index_details`, so the remote client deserializes it back into the same type instead of reconstructing from a bespoke sharding-spec JSON shape. Co-Authored-By: Claude Opus 4.8 --- rust/lancedb/src/remote/table.rs | 115 +++++++------------------------ 1 file changed, 24 insertions(+), 91 deletions(-) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index b3003a626..be3fcf5b8 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1489,85 +1489,6 @@ impl RemoteTable { } } -/// Reconstruct an [`LsmWriteSpec`](crate::table::LsmWriteSpec) from the MemWAL -/// index's `details` JSON as emitted by the server's `describe_indices`. -/// -/// Mirrors the native reconstruction in `table::merge::lsm`, but reads the -/// server-resolved `column` name from the sharding field rather than a Lance -/// field id — field ids do not travel to the remote client, so the server -/// resolves the name against its schema before serializing the details. -fn lsm_write_spec_from_details_json( - details: &str, -) -> std::result::Result { - use crate::table::LsmWriteSpec; - - #[derive(Deserialize)] - struct Details { - #[serde(default)] - sharding_specs: Vec, - #[serde(default)] - maintained_indexes: Vec, - #[serde(default)] - writer_config_defaults: HashMap, - } - #[derive(Deserialize)] - struct ShardingSpec { - #[serde(default)] - fields: Vec, - } - #[derive(Deserialize)] - struct ShardingField { - transform: Option, - /// Server-resolved routing column name (bucket / identity only). - #[serde(default)] - column: Option, - #[serde(default)] - parameters: HashMap, - } - - let details: Details = serde_json::from_str(details) - .map_err(|e| format!("failed to parse MemWAL index details: {}", e))?; - - let field = details - .sharding_specs - .first() - .and_then(|s| s.fields.first()) - .ok_or_else(|| "MemWAL index details has no sharding field".to_string())?; - - let column = || { - field - .column - .clone() - .ok_or_else(|| "MemWAL sharding field is missing the resolved column name".to_string()) - }; - - let base = match field.transform.as_deref() { - Some("bucket") => { - let num_buckets = field - .parameters - .get("num_buckets") - .and_then(|s| s.parse::().ok()) - .filter(|n| *n > 0) - .ok_or_else(|| { - "MemWAL bucket spec has a missing or invalid num_buckets".to_string() - })?; - LsmWriteSpec::bucket(column()?, num_buckets) - } - Some("identity") => LsmWriteSpec::identity(column()?), - Some("unsharded") => LsmWriteSpec::unsharded(), - other => { - return Err(format!( - "MemWAL index has an unsupported sharding transform {:?}", - other - )); - } - }; - - Ok(base - .with_maintained_indexes(details.maintained_indexes) - .with_writer_config_defaults(details.writer_config_defaults)) -} - #[async_trait] impl BaseTable for RemoteTable { fn as_any(&self) -> &dyn std::any::Any { @@ -2361,8 +2282,8 @@ impl BaseTable for RemoteTable { // The spec lives in the `__lance_mem_wal` system index, which the // curated `index/list` surface filters out by default. `include_system` // opts that entry back in for this read only — `list_indices` stays - // curated. The system index's `index_details` carries the decoded - // MemWAL state (with the server-resolved shard column name). + // curated. The server serializes the resolved `LsmWriteSpec` (shard + // column already mapped from its field id) into `index_details`. let mut request = self.post_read(&format!("/v1/table/{}/index/list/", self.identifier)); let version = self.current_version().await; let mut body = serde_json::json!({ "version": version, "include_system": true }); @@ -2408,11 +2329,18 @@ impl BaseTable for RemoteTable { status_code: None, })?; - let spec = lsm_write_spec_from_details_json(&details).map_err(|msg| Error::Http { - source: msg.into(), - request_id, - status_code: None, - })?; + // The server serializes the resolved `LsmWriteSpec` directly, so it + // deserializes back into the same type — no client-side reconstruction. + let spec: crate::table::LsmWriteSpec = + serde_json::from_str(&details).map_err(|e| Error::Http { + source: format!( + "Failed to parse MemWAL index details as LsmWriteSpec: {}", + e + ) + .into(), + request_id, + status_code: None, + })?; Ok(Some(spec)) } @@ -5465,11 +5393,16 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!(body["include_system"], serde_json::json!(true)); - // The MemWAL index carries the decoded state in `index_details`, - // including the server-resolved `column` name. A user index is - // included too, with a type the client has no variant for — proving - // the getter reads by name and ignores `index_type`. - let details = r#"{"sharding_specs":[{"fields":[{"transform":"bucket","column":"id","parameters":{"num_buckets":"4"}}]}],"maintained_indexes":["id_idx"],"writer_config_defaults":{"durable_write":"false"}}"#; + // The server serializes the resolved `LsmWriteSpec` into + // `index_details`. A user index is included too, with a type the + // client has no variant for — proving the getter reads by name and + // ignores `index_type`. + let details = serde_json::to_string( + &crate::table::LsmWriteSpec::bucket("id", 4) + .with_maintained_indexes(["id_idx"]) + .with_writer_config_defaults([("durable_write", "false")]), + ) + .unwrap(); let response = serde_json::json!({ "indexes": [ { "index_name": "v_idx", "index_type": "BTREE", "columns": ["v"] },