feat: add get_lsm_write_spec to read the installed LSM write spec

Adds `Table::get_lsm_write_spec` returning `Option<LsmWriteSpec>`, the read
counterpart to `set_lsm_write_spec` / `unset_lsm_write_spec`.

- Rust core: `NativeTable` reconstructs the spec from the MemWAL index
  details (`mem_wal_index_details`), resolving the shard column via the
  dataset schema. Read-only, so unaffected by describe_indices filtering.
- Remote: `RemoteTable` reads the `__lance_mem_wal` system index via
  `index/list` with `include_system: true` (kept out of the curated
  `list_indices` surface) and parses its `details` JSON, using the
  server-resolved `column` name.
- Python and TypeScript bindings, sync + async, with round-trip tests.

The remote path depends on lance emitting the resolved shard column name in
the MemWAL index details JSON, and on the server honoring `include_system`
on `index/list`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-07-07 14:45:33 -05:00
co-authored by Claude Opus 4.8
parent 70126943ff
commit 6633f83f7b
11 changed files with 604 additions and 1 deletions
+40
View File
@@ -2992,6 +2992,46 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
}),
).rejects.toThrow();
});
it("reads back the installed spec via getLsmWriteSpec", async () => {
const conn = await connect(tmpDir.name);
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
// Nothing installed yet.
expect(await table.getLsmWriteSpec()).toBeUndefined();
// Bucket spec round-trips, including writer config defaults.
await table.setLsmWriteSpec({
specType: "bucket",
column: "id",
numBuckets: 4,
writerConfigDefaults: { durable_write: "false" },
});
const spec = await table.getLsmWriteSpec();
expect(spec).toBeDefined();
expect(spec?.specType).toBe("bucket");
expect(spec?.column).toBe("id");
expect(spec?.numBuckets).toBe(4);
expect(spec?.writerConfigDefaults).toEqual({ durable_write: "false" });
// After unset, undefined again.
await table.unsetLsmWriteSpec();
expect(await table.getLsmWriteSpec()).toBeUndefined();
// Identity round-trips (column recovered from the schema).
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
const identity = await table.getLsmWriteSpec();
expect(identity?.specType).toBe("identity");
expect(identity?.column).toBe("id");
await table.unsetLsmWriteSpec();
// Unsharded round-trips (no routing column).
await table.setLsmWriteSpec({ specType: "unsharded" });
const unsharded = await table.getLsmWriteSpec();
expect(unsharded?.specType).toBe("unsharded");
expect(unsharded?.column).toBeFalsy();
});
});
describe("LSM merge insert", () => {
+15
View File
@@ -585,6 +585,17 @@ export abstract class Table {
* @returns {Promise<void>}
*/
abstract unsetLsmWriteSpec(): Promise<void>;
/**
* Read the {@link LsmWriteSpec} currently installed on this table.
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* The returned spec — including its `maintainedIndexes` and
* `writerConfigDefaults` — mirrors what was passed to
* {@link Table#setLsmWriteSpec}.
* @returns {Promise<LsmWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
/**
* Drain and close any cached MemWAL shard writers held for this table.
*
@@ -1091,6 +1102,10 @@ export class LocalTable extends Table {
return await this.inner.unsetLsmWriteSpec();
}
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
return (await this.inner.getLsmWriteSpec()) ?? undefined;
}
async closeLsmWriters(): Promise<void> {
return await this.inner.closeLsmWriters();
}
+51
View File
@@ -411,6 +411,16 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
let spec = self
.inner_ref()?
.get_lsm_write_spec()
.await
.default_error()?;
Ok(spec.map(LsmWriteSpec::from))
}
#[napi(catch_unwind)]
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
self.inner_ref()?.close_lsm_writers().await.default_error()
@@ -728,6 +738,47 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
use lancedb::table::LsmWriteSpec as Native;
match spec {
Native::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "bucket".to_string(),
column: Some(column),
num_buckets: Some(num_buckets),
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Identity {
column,
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "identity".to_string(),
column: Some(column),
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Unsharded {
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "unsharded".to_string(),
column: None,
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
}
}
}
/// Statistics about a compaction operation.
#[napi(object)]
#[derive(Clone, Debug)]
+1
View File
@@ -226,6 +226,7 @@ class Table:
async def set_unenforced_primary_key(self, columns: List[str]) -> None: ...
async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
async def unset_lsm_write_spec(self) -> None: ...
async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
async def close_lsm_writers(self) -> None: ...
@property
def tags(self) -> Tags: ...
+4
View File
@@ -912,6 +912,10 @@ class RemoteTable(Table):
"""Not supported on LanceDB Cloud."""
return LOOP.run(self._table.unset_lsm_write_spec())
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the installed LsmWriteSpec, or ``None``."""
return LOOP.run(self._table.get_lsm_write_spec())
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
+16
View File
@@ -3742,6 +3742,11 @@ class LanceTable(Table):
[`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec]."""
return LOOP.run(self._table.unset_lsm_write_spec())
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the installed LsmWriteSpec, or ``None``. See
[`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
return LOOP.run(self._table.get_lsm_write_spec())
def close_lsm_writers(self) -> None:
"""Close cached MemWAL shard writers. See
[`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
@@ -4391,6 +4396,17 @@ class AsyncTable:
"""
await self._inner.unset_lsm_write_spec()
async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the LsmWriteSpec currently installed on this table.
Returns ``None`` when the MemWAL LSM write path is not enabled (no
spec has been set, or it was removed with `unset_lsm_write_spec`).
The returned spec — including its ``maintained_indexes`` and
``writer_config_defaults`` — mirrors what was passed to
`set_lsm_write_spec`.
"""
return await self._inner.get_lsm_write_spec()
async def close_lsm_writers(self) -> None:
"""Drain and close any cached MemWAL shard writers for this table.
@@ -136,3 +136,62 @@ def test_lsm_write_spec_identity_and_writer_config_defaults():
s = s.with_writer_config_defaults({"durable_write": "false"})
assert s.writer_config_defaults == {"durable_write": "false"}
assert "durable_write" in repr(s)
def test_get_lsm_write_spec(tmp_path):
_db, table = _make_table(tmp_path)
table.set_unenforced_primary_key("id")
# None when nothing is installed.
assert table.get_lsm_write_spec() is None
# Bucket spec round-trips, including writer config defaults.
table.set_lsm_write_spec(
LsmWriteSpec.bucket("id", 4).with_writer_config_defaults(
{"durable_write": "false"}
)
)
spec = table.get_lsm_write_spec()
assert spec is not None
assert spec.spec_type == "bucket"
assert spec.column == "id"
assert spec.num_buckets == 4
assert spec.writer_config_defaults == {"durable_write": "false"}
# After unset, None again.
table.unset_lsm_write_spec()
assert table.get_lsm_write_spec() is None
# Identity round-trips (column recovered from the schema).
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
spec = table.get_lsm_write_spec()
assert spec.spec_type == "identity"
assert spec.column == "id"
table.unset_lsm_write_spec()
# Unsharded round-trips (no routing column).
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
spec = table.get_lsm_write_spec()
assert spec.spec_type == "unsharded"
assert spec.column is None
@pytest.mark.asyncio
async def test_async_get_lsm_write_spec(tmp_path):
db = await lancedb.connect_async(
tmp_path, read_consistency_interval=timedelta(seconds=0)
)
table = await db.create_table(
"t",
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
)
assert await table.get_lsm_write_spec() is None
await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
spec = await table.get_lsm_write_spec()
assert spec is not None
assert spec.spec_type == "bucket"
assert spec.column == "id"
assert spec.num_buckets == 8
await table.unset_lsm_write_spec()
assert await table.get_lsm_write_spec() is None
+14
View File
@@ -322,6 +322,12 @@ impl From<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(inner: lancedb::table::LsmWriteSpec) -> Self {
Self { inner }
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct AddColumnsResult {
@@ -1029,6 +1035,14 @@ impl Table {
})
}
pub fn get_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let spec = inner.get_lsm_write_spec().await.infer_error()?;
Ok(spec.map(LsmWriteSpec::from))
})
}
pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
+214
View File
@@ -1489,6 +1489,85 @@ impl<S: HttpSend + 'static> RemoteTable<S> {
}
}
/// Reconstruct an [`LsmWriteSpec`](crate::table::LsmWriteSpec) from the MemWAL
/// index's `details` JSON as emitted by the server's `describe_indices`.
///
/// Mirrors the native reconstruction in `table::merge::lsm`, but reads the
/// server-resolved `column` name from the sharding field rather than a Lance
/// field id — field ids do not travel to the remote client, so the server
/// resolves the name against its schema before serializing the details.
fn lsm_write_spec_from_details_json(
details: &str,
) -> std::result::Result<crate::table::LsmWriteSpec, String> {
use crate::table::LsmWriteSpec;
#[derive(Deserialize)]
struct Details {
#[serde(default)]
sharding_specs: Vec<ShardingSpec>,
#[serde(default)]
maintained_indexes: Vec<String>,
#[serde(default)]
writer_config_defaults: HashMap<String, String>,
}
#[derive(Deserialize)]
struct ShardingSpec {
#[serde(default)]
fields: Vec<ShardingField>,
}
#[derive(Deserialize)]
struct ShardingField {
transform: Option<String>,
/// Server-resolved routing column name (bucket / identity only).
#[serde(default)]
column: Option<String>,
#[serde(default)]
parameters: HashMap<String, String>,
}
let details: Details = serde_json::from_str(details)
.map_err(|e| format!("failed to parse MemWAL index details: {}", e))?;
let field = details
.sharding_specs
.first()
.and_then(|s| s.fields.first())
.ok_or_else(|| "MemWAL index details has no sharding field".to_string())?;
let column = || {
field
.column
.clone()
.ok_or_else(|| "MemWAL sharding field is missing the resolved column name".to_string())
};
let base = match field.transform.as_deref() {
Some("bucket") => {
let num_buckets = field
.parameters
.get("num_buckets")
.and_then(|s| s.parse::<u32>().ok())
.filter(|n| *n > 0)
.ok_or_else(|| {
"MemWAL bucket spec has a missing or invalid num_buckets".to_string()
})?;
LsmWriteSpec::bucket(column()?, num_buckets)
}
Some("identity") => LsmWriteSpec::identity(column()?),
Some("unsharded") => LsmWriteSpec::unsharded(),
other => {
return Err(format!(
"MemWAL index has an unsupported sharding transform {:?}",
other
));
}
};
Ok(base
.with_maintained_indexes(details.maintained_indexes)
.with_writer_config_defaults(details.writer_config_defaults))
}
#[async_trait]
impl<S: HttpSend> BaseTable for RemoteTable<S> {
fn as_any(&self) -> &dyn std::any::Any {
@@ -2278,6 +2357,65 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
Ok(())
}
async fn get_lsm_write_spec(&self) -> Result<Option<crate::table::LsmWriteSpec>> {
// The spec lives in the `__lance_mem_wal` system index, which the
// curated `index/list` surface filters out by default. `include_system`
// opts that entry back in for this read only — `list_indices` stays
// curated. The system index's `index_details` carries the decoded
// MemWAL state (with the server-resolved shard column name).
let mut request = self.post_read(&format!("/v1/table/{}/index/list/", self.identifier));
let version = self.current_version().await;
let mut body = serde_json::json!({ "version": version, "include_system": true });
self.apply_branch_body(&mut body);
request = request.json(&body);
let (request_id, response) = self.send(request, true).await?;
let response = self.check_table_response(&request_id, response).await?;
let body = response.text().await.err_to_http(request_id.clone())?;
#[derive(Deserialize)]
struct IndexList {
#[serde(default)]
indexes: Vec<IndexEntry>,
}
// Only the name and opaque details are read; `index_type` is ignored so
// the system index's type (which has no client `IndexType` variant) does
// not break deserialization.
#[derive(Deserialize)]
struct IndexEntry {
index_name: String,
index_details: Option<String>,
}
let parsed: IndexList = serde_json::from_str(&body).map_err(|e| Error::Http {
source: format!("Failed to parse index list response: {}", e).into(),
request_id: request_id.clone(),
status_code: None,
})?;
let Some(entry) = parsed
.indexes
.into_iter()
.find(|i| i.index_name.as_str() == lance_index::mem_wal::MEM_WAL_INDEX_NAME)
else {
// No MemWAL index installed → LSM write path not enabled.
return Ok(None);
};
let details = entry.index_details.ok_or_else(|| Error::Http {
source: "index/list returned the MemWAL index without details".into(),
request_id: request_id.clone(),
status_code: None,
})?;
let spec = lsm_write_spec_from_details_json(&details).map_err(|msg| Error::Http {
source: msg.into(),
request_id,
status_code: None,
})?;
Ok(Some(spec))
}
async fn tags(&self) -> Result<Box<dyn Tags + '_>> {
Ok(Box::new(RemoteTags { inner: self }))
}
@@ -5317,6 +5455,82 @@ mod tests {
table.unset_lsm_write_spec().await.unwrap();
}
#[tokio::test]
async fn test_get_lsm_write_spec() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/index/list/");
// The getter opts the system index back into the listing.
let body = request.body().unwrap().as_bytes().unwrap();
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["include_system"], serde_json::json!(true));
// The MemWAL index carries the decoded state in `index_details`,
// including the server-resolved `column` name. A user index is
// included too, with a type the client has no variant for — proving
// the getter reads by name and ignores `index_type`.
let details = r#"{"sharding_specs":[{"fields":[{"transform":"bucket","column":"id","parameters":{"num_buckets":"4"}}]}],"maintained_indexes":["id_idx"],"writer_config_defaults":{"durable_write":"false"}}"#;
let response = serde_json::json!({
"indexes": [
{ "index_name": "v_idx", "index_type": "BTREE", "columns": ["v"] },
{
"index_name": "__lance_mem_wal",
"index_type": "MemWal",
"columns": [],
"index_details": details,
},
]
});
http::Response::builder()
.status(200)
.body(response.to_string())
.unwrap()
});
let spec = table
.get_lsm_write_spec()
.await
.unwrap()
.expect("a spec should be reported");
match spec {
crate::table::LsmWriteSpec::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => {
assert_eq!(column, "id");
assert_eq!(num_buckets, 4);
assert_eq!(maintained_indexes, vec!["id_idx".to_string()]);
assert_eq!(
writer_config_defaults
.get("durable_write")
.map(String::as_str),
Some("false")
);
}
other => panic!("expected a bucket spec, got {:?}", other),
}
}
#[tokio::test]
async fn test_get_lsm_write_spec_absent() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/index/list/");
// No __lance_mem_wal entry → the LSM write path is not enabled.
let response = serde_json::json!({
"indexes": [
{ "index_name": "v_idx", "index_type": "BTREE", "columns": ["v"] },
]
});
http::Response::builder()
.status(200)
.body(response.to_string())
.unwrap()
});
assert!(table.get_lsm_write_spec().await.unwrap().is_none());
}
#[tokio::test]
async fn test_wait_for_index() {
let table = _make_table_with_indices(0);
+98
View File
@@ -581,6 +581,16 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
message: "unset_lsm_write_spec is not supported on this table type".into(),
})
}
/// Read the [`LsmWriteSpec`] currently installed on this table, returning
/// `None` when the MemWAL LSM write path is not enabled.
///
/// The default implementation returns `NotSupported`. Implementations that
/// support the MemWAL LSM write path must override this.
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
Err(Error::NotSupported {
message: "get_lsm_write_spec is not supported on this table type".into(),
})
}
/// Drain and close any cached MemWAL shard writers for this table.
///
/// The default implementation is a no-op; table types that maintain
@@ -1538,6 +1548,29 @@ impl Table {
self.inner.unset_lsm_write_spec().await
}
/// Read the [`LsmWriteSpec`] currently installed on this table.
///
/// Returns `Ok(None)` when the MemWAL LSM write path is not enabled (no
/// spec has been set, or it was removed with [`Table::unset_lsm_write_spec`]).
/// The returned spec — including its [`LsmWriteSpec::maintained_indexes`] and
/// [`LsmWriteSpec::writer_config_defaults`] — mirrors what was passed to
/// [`Table::set_lsm_write_spec`].
///
/// # Example
///
/// ```
/// # use lancedb::table::Table;
/// # async fn example(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// if let Some(spec) = table.get_lsm_write_spec().await? {
/// println!("LSM write path enabled: {:?}", spec);
/// }
/// # Ok(())
/// # }
/// ```
pub async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
self.inner.get_lsm_write_spec().await
}
/// Drain and close any cached MemWAL shard writers held for this table.
///
/// When an [`LsmWriteSpec`] is installed, `merge_insert` opens MemWAL shard
@@ -2850,6 +2883,10 @@ impl BaseTable for NativeTable {
merge::lsm::unset_lsm_write_spec(self).await
}
async fn get_lsm_write_spec(&self) -> Result<Option<LsmWriteSpec>> {
merge::lsm::get_lsm_write_spec(self).await
}
async fn close_lsm_writers(&self) -> Result<()> {
merge::lsm::close_lsm_writers(self).await
}
@@ -4411,6 +4448,67 @@ mod tests {
}
}
#[tokio::test]
async fn test_get_lsm_write_spec() {
let tmp_dir = tempdir().unwrap();
let uri = tmp_dir.path().to_str().unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("region", DataType::Utf8, true),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(arrow_array::Int64Array::from(vec![1, 2, 3])),
Arc::new(StringArray::from(vec!["a", "b", "c"])),
],
)
.unwrap();
let reader: Box<dyn arrow_array::RecordBatchReader + Send> =
Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema.clone()));
let conn = ConnectBuilder::new(uri)
.read_consistency_interval(Duration::from_secs(0))
.execute()
.await
.unwrap();
let table = conn.create_table("t", reader).execute().await.unwrap();
// No spec installed yet.
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// A real scalar index is needed to name it as a maintained index.
table
.create_index(&["id"], Index::Auto)
.execute()
.await
.unwrap();
let idx_name = table.list_indices().await.unwrap()[0].name.clone();
// Bucket spec round-trips exactly, including the routing column (recovered
// from its field id), maintained indexes, and writer config defaults.
let spec = LsmWriteSpec::bucket("id", 4)
.with_maintained_indexes([idx_name])
.with_writer_config_defaults([("durable_write", "false")]);
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
// After unset, no spec is reported.
table.unset_lsm_write_spec().await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), None);
// Identity sharding round-trips (column recovered from the schema).
let spec = LsmWriteSpec::identity("region");
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
table.unset_lsm_write_spec().await.unwrap();
// Unsharded round-trips (no routing column).
let spec = LsmWriteSpec::unsharded();
table.set_lsm_write_spec(spec.clone()).await.unwrap();
assert_eq!(table.get_lsm_write_spec().await.unwrap(), Some(spec));
}
#[tokio::test]
pub async fn test_stats() {
let tmp_dir = tempdir().unwrap();
+92 -1
View File
@@ -32,7 +32,7 @@ use lance::dataset::mem_wal::{
};
use lance::index::DatasetIndexExt;
use lance_core::datatypes::Schema as LanceSchema;
use lance_index::mem_wal::{MemWalIndexDetails, ShardingSpec};
use lance_index::mem_wal::{MemWalIndexDetails, ShardingField, ShardingSpec};
use tokio::sync::RwLock;
use uuid::Uuid;
@@ -148,6 +148,97 @@ pub(crate) async fn unset_lsm_write_spec(table: &NativeTable) -> Result<()> {
Ok(())
}
// =============================================================================
// get_lsm_write_spec
// =============================================================================
/// Read the [`LsmWriteSpec`] currently installed on the table, if any.
///
/// Reconstructs the spec from the MemWAL index's stored details — the same
/// metadata [`set_lsm_write_spec`] writes — read directly via
/// [`mem_wal_index_details`](DatasetMemWalExt::mem_wal_index_details) (the raw
/// manifest record, not the `describe_indices` enrichment path, so it is
/// unaffected by system-index filtering there). Returns `Ok(None)` when no
/// MemWAL index is installed.
#[allow(clippy::redundant_pub_crate)]
pub(crate) async fn get_lsm_write_spec(table: &NativeTable) -> Result<Option<LsmWriteSpec>> {
let dataset = table.dataset.get().await?;
let Some(details) = dataset.mem_wal_index_details().await? else {
return Ok(None);
};
let spec = lsm_write_spec_from_details(&details, dataset.schema())?;
Ok(Some(spec))
}
/// Reconstruct the public [`LsmWriteSpec`] from stored MemWAL index details.
///
/// The sharding transform and its parameters recover the mode; the routing
/// column is resolved from the sharding field's source id back to its schema
/// name (only bucket / identity carry a column). `maintained_indexes` and
/// `writer_config_defaults` are copied verbatim, so the result round-trips
/// what `set_lsm_write_spec` installed.
fn lsm_write_spec_from_details(
details: &MemWalIndexDetails,
schema: &LanceSchema,
) -> Result<LsmWriteSpec> {
let spec = details
.sharding_specs
.first()
.ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: MemWAL index has no sharding spec".to_string(),
})?;
let field = spec.fields.first().ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: MemWAL index has an empty sharding spec".to_string(),
})?;
let base = match field.transform.as_deref() {
Some(BUCKET_TRANSFORM) => {
let num_buckets = field
.parameters
.get(NUM_BUCKETS_PARAM)
.and_then(|s| s.parse::<u32>().ok())
.filter(|n| *n > 0)
.ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: MemWAL bucket spec has a missing or invalid num_buckets parameter".to_string(),
})?;
LsmWriteSpec::bucket(sharding_column(field, schema)?, num_buckets)
}
Some(IDENTITY_TRANSFORM) => LsmWriteSpec::identity(sharding_column(field, schema)?),
Some(UNSHARDED_TRANSFORM) => LsmWriteSpec::unsharded(),
other => {
return Err(Error::Runtime {
message: format!(
"get_lsm_write_spec: MemWAL index has an unsupported sharding transform {:?}",
other
),
});
}
};
Ok(base
.with_maintained_indexes(details.maintained_indexes.clone())
.with_writer_config_defaults(details.writer_config_defaults.clone()))
}
/// Resolve the single routing column name from a sharding field's source id.
///
/// `set_lsm_write_spec` records the shard column by its Lance field id, so the
/// name is recovered by looking that id up in the current schema.
fn sharding_column(field: &ShardingField, schema: &LanceSchema) -> Result<String> {
let source_id = *field.source_ids.first().ok_or_else(|| Error::Runtime {
message: "get_lsm_write_spec: sharding field has no source column".to_string(),
})?;
schema
.field_by_id(source_id)
.map(|f| f.name.clone())
.ok_or_else(|| Error::Runtime {
message: format!(
"get_lsm_write_spec: sharding source field id {} not found in schema",
source_id
),
})
}
// =============================================================================
// close_lsm_writers
// =============================================================================