refactor!: rename MemWAL APIs to RTE (realtime engine) (#4598)

## What Renames the user-facing MemWAL surface to **RTE** (realtime engine) across Rust, Python, TypeScript, and Java, including the REST wire format. This follows the LSM → MemWAL rename in #4075. | Before | After | |---|---| | `MemWalWriteSpec` / `MemWalStats` | `RTEWriteSpec` / `RTEStats` | | `enable_memwal` / `disable_memwal` / `get_memwal_spec` | `enable_rte` / `disable_rte` / `get_rte_spec` | | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | `flush_rte` / `compact_rte` / `checkpoint_rte` | | `get_memwal_stats` / `close_memwal_writers` | `get_rte_stats` / `close_rte_writers` | | `use_memwal` (query, merge insert) | `use_rte` | | TS/Java `enableMemWal`, `useMemWal`, ... | `enableRTE`, `useRTE`, ... (napi `js_name` set explicitly) | | Java `LanceDbTableMemWal` | `LanceDbTableRTE` | | `table/memwal_stats.rs` | `table/rte_stats.rs` | Docs and comments now say "realtime engine (RTE)". Text about Lance's underlying mechanism keeps "MemWAL" ("Lance's MemWAL", "MemWAL index"). **Left alone:** Lance-owned names (`MemWalIndexDetails`, `DatasetMemWalExt`, `InitializeMemWal*`, `mem_wal` paths), internal `Lsm*` names and modules, and the `does not support with_row_id` refusal text that the hybrid fallback matches on. ## Compatibility Breaking, for an experimental API. The wire format changes, so this must roll out together with the matching sophon change (lancedb/sophon#8870): - Routes: `/enable_memwal/` → `/enable_rte/`, `/disable_memwal/` → `/disable_rte/`, `/get_memwal_spec/` → `/get_rte_spec/`, `/get_memwal_stats/` → `/get_rte_stats/`, `/flush_memwal/` → `/flush_rte/`, `/compact_memwal/` → `/compact_rte/` - Response keys: `memwal_spec` → `rte_spec`, `memwal_stats` → `rte_stats` - Request field: `use_memwal` → `use_rte` on query and merge-insert bodies The SQL keyword becomes `ALTER TABLE ... SET RTE` server-side. The guides under `docs/web` will be updated separately. ## Validation - Rust: `cargo clippy --features remote --tests --examples` clean; `cargo test --features remote -p lancedb --lib`: 1533 passed - Java: `mvn test`: 37 passed (Spotless applied with JDK 11) - Python: `ruff format` / `ruff check` clean; targeted tests (`test_rte_spec`, `test_merge_insert_lsm`, `test_remote_db`, `test_wal_hybrid`, `test_query`, `test_permutation`, `test_table`): 708 passed - TypeScript: `pnpm build`; `table`, `query` tests: 606 passed; `biome check` clean; `docs/src/js` regenerated with `pnpm run docs` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-23 12:12:27 +00:00 · 2026-10-20 11:58:57 -05:00
parent 1772f4ef83
commit 92825f6f13
73 changed files with 1656 additions and 1600 deletions
@@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
 use session::Session;
 use table::{
    AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
-    FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MemWalWriteSpec, MergeResult,
-    PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table,
+    FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MergeResult, PyBlobFile,
+    RTEWriteSpec, RefreshColumnResult, RefreshMaterializedViewResult, Table,
    UpdateFieldMetadataResult, UpdateResult,
 };

@@ -104,7 +104,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
    m.add_class::<UpdateFieldMetadataResult>()?;
    m.add_class::<AddResult>()?;
    m.add_class::<MergeResult>()?;
-    m.add_class::<MemWalWriteSpec>()?;
+    m.add_class::<RTEWriteSpec>()?;
    m.add_class::<DeleteResult>()?;
    m.add_class::<DropColumnsResult>()?;
    m.add_class::<UpdateResult>()?;
@@ -333,7 +333,7 @@ pub struct PyQueryRequest {
    pub select_source_columns: Option<HashMap<String, String>>,
    pub fast_search: Option<bool>,
    pub with_row_id: Option<bool>,
-    pub use_memwal: Option<bool>,
+    pub use_rte: Option<bool>,
    pub column: Option<String>,
    pub query_vector: Option<PyQueryVectors>,
    pub nprobes: Option<usize>,
@@ -366,7 +366,7 @@ impl From<AnyQuery> for PyQueryRequest {
                select: PySelect(query_request.select),
                fast_search: Some(query_request.fast_search),
                with_row_id: Some(query_request.with_row_id),
-                use_memwal: query_request.use_memwal,
+                use_rte: query_request.use_rte,
                column: None,
                query_vector: None,
                nprobes: None,
@@ -395,7 +395,7 @@ impl From<AnyQuery> for PyQueryRequest {
                select: PySelect(vector_query.base.select),
                fast_search: Some(vector_query.base.fast_search),
                with_row_id: Some(vector_query.base.with_row_id),
-                use_memwal: vector_query.base.use_memwal,
+                use_rte: vector_query.base.use_rte,
                column: vector_query.column,
                query_vector: Some(PyQueryVectors(vector_query.query_vector)),
                nprobes: vector_query.nprobes,
@@ -540,8 +540,8 @@ impl Query {
        self.inner = self.inner.clone().fast_search();
    }

-    pub fn use_memwal(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_memwal(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
    }

    pub fn with_row_id(&mut self) {
@@ -706,8 +706,8 @@ impl TakeQuery {
        self.inner = self.inner.clone().with_row_id();
    }

-    pub fn use_memwal(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_memwal(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
    }

    #[pyo3(signature = ())]
@@ -819,8 +819,8 @@ impl FTSQuery {
        self.inner = self.inner.clone().fast_search();
    }

-    pub fn use_memwal(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_memwal(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
    }

    pub fn with_row_id(&mut self) {
@@ -970,8 +970,8 @@ impl VectorQuery {
        self.inner = self.inner.clone().fast_search();
    }

-    pub fn use_memwal(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_memwal(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
    }

    pub fn with_row_id(&mut self) {
@@ -1178,9 +1178,9 @@ impl HybridQuery {
        self.inner_fts.postfilter();
    }

-    pub fn use_memwal(&mut self, enable: bool) {
-        self.inner_vec.use_memwal(enable);
-        self.inner_fts.use_memwal(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner_vec.use_rte(enable);
+        self.inner_fts.use_rte(enable);
    }

    pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> {
@@ -39,13 +39,13 @@ use pyo3::{

 mod scannable;

-/// Convert `MemWalStats` to a Python dict, preserving the per-table-shard list.
+/// Convert `RTEStats` to a Python dict, preserving the per-table-shard list.
 ///
 /// Deliberately not flattened to a table-level summary: a table is N
 /// table shards on one node, and the per-shard detail is the reason the
 /// endpoint exists — flattening hides the single hot table shard someone opened
 /// it to find.
-fn memwal_stats_to_py(py: Python<'_>, stats: &lancedb::table::MemWalStats) -> PyResult<Py<PyDict>> {
+fn rte_stats_to_py(py: Python<'_>, stats: &lancedb::table::RTEStats) -> PyResult<Py<PyDict>> {
    let out = PyDict::new(py);
    let table_shards = PyList::empty(py);
    for b in &stats.table_shards {
@@ -261,8 +261,8 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
    }
 }

-/// Specification selecting Lance's MemWAL LSM-style write path for
-/// `merge_insert`.
+/// Specification enabling the realtime engine (RTE), which routes
+/// `merge_insert` through Lance's MemWAL LSM-style write path.
 ///
 /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
 /// classmethods, then optionally chain `with_maintained_indexes(...)` and
@@ -270,17 +270,17 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
 /// table has, including ones created later.
 #[pyclass(module = "lancedb._lancedb", from_py_object)]
 #[derive(Clone, Debug)]
-pub struct MemWalWriteSpec {
-    inner: lancedb::table::MemWalWriteSpec,
+pub struct RTEWriteSpec {
+    inner: lancedb::table::RTEWriteSpec,
 }

 #[pymethods]
-impl MemWalWriteSpec {
+impl RTEWriteSpec {
    /// Hash-bucket sharding by the unenforced primary key column.
    #[staticmethod]
    pub fn bucket(column: String, num_buckets: u32) -> Self {
        Self {
-            inner: lancedb::table::MemWalWriteSpec::bucket(column, num_buckets),
+            inner: lancedb::table::RTEWriteSpec::bucket(column, num_buckets),
        }
    }

@@ -294,20 +294,20 @@ impl MemWalWriteSpec {
    #[staticmethod]
    pub fn identity(column: String) -> Self {
        Self {
-            inner: lancedb::table::MemWalWriteSpec::identity(column),
+            inner: lancedb::table::RTEWriteSpec::identity(column),
        }
    }

    /// No sharding — every `merge_insert` call writes to a single
-    /// MemWAL shard.
+    /// RTE shard.
    #[staticmethod]
    pub fn unsharded() -> Self {
        Self {
-            inner: lancedb::table::MemWalWriteSpec::unsharded(),
+            inner: lancedb::table::RTEWriteSpec::unsharded(),
        }
    }

-    /// Set which indexes the MemWAL maintains. `None` (the default) is
+    /// Set which indexes the RTE maintains. `None` (the default) is
    /// every index the table has, including ones created later; a list is
    /// verbatim, and an empty list maintains nothing.
    #[pyo3(signature = (indexes))]
@@ -327,33 +327,33 @@ impl MemWalWriteSpec {

    pub fn __repr__(&self) -> String {
        match &self.inner {
-            lancedb::table::MemWalWriteSpec::Bucket {
+            lancedb::table::RTEWriteSpec::Bucket {
                column,
                num_buckets,
                maintained_indexes,
                writer_config_defaults,
            } => format!(
-                "MemWalWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
+                "RTEWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
                column,
                num_buckets,
                fmt_maintained(maintained_indexes),
                writer_config_defaults,
            ),
-            lancedb::table::MemWalWriteSpec::Identity {
+            lancedb::table::RTEWriteSpec::Identity {
                column,
                maintained_indexes,
                writer_config_defaults,
            } => format!(
-                "MemWalWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
+                "RTEWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
                column,
                fmt_maintained(maintained_indexes),
                writer_config_defaults,
            ),
-            lancedb::table::MemWalWriteSpec::Unsharded {
+            lancedb::table::RTEWriteSpec::Unsharded {
                maintained_indexes,
                writer_config_defaults,
            } => format!(
-                "MemWalWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
+                "RTEWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
                fmt_maintained(maintained_indexes),
                writer_config_defaults,
            ),
@@ -365,9 +365,9 @@ impl MemWalWriteSpec {
    #[getter]
    pub fn spec_type(&self) -> &'static str {
        match &self.inner {
-            lancedb::table::MemWalWriteSpec::Bucket { .. } => "bucket",
-            lancedb::table::MemWalWriteSpec::Identity { .. } => "identity",
-            lancedb::table::MemWalWriteSpec::Unsharded { .. } => "unsharded",
+            lancedb::table::RTEWriteSpec::Bucket { .. } => "bucket",
+            lancedb::table::RTEWriteSpec::Identity { .. } => "identity",
+            lancedb::table::RTEWriteSpec::Unsharded { .. } => "unsharded",
        }
    }

@@ -375,9 +375,9 @@ impl MemWalWriteSpec {
    #[getter]
    pub fn column(&self) -> Option<String> {
        match &self.inner {
-            lancedb::table::MemWalWriteSpec::Bucket { column, .. }
-            | lancedb::table::MemWalWriteSpec::Identity { column, .. } => Some(column.clone()),
-            lancedb::table::MemWalWriteSpec::Unsharded { .. } => None,
+            lancedb::table::RTEWriteSpec::Bucket { column, .. }
+            | lancedb::table::RTEWriteSpec::Identity { column, .. } => Some(column.clone()),
+            lancedb::table::RTEWriteSpec::Unsharded { .. } => None,
        }
    }

@@ -385,12 +385,12 @@ impl MemWalWriteSpec {
    #[getter]
    pub fn num_buckets(&self) -> Option<u32> {
        match &self.inner {
-            lancedb::table::MemWalWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
+            lancedb::table::RTEWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
            _ => None,
        }
    }

-    /// Indexes the MemWAL keeps up to date, or `None` for every supported one.
+    /// Indexes the RTE keeps up to date, or `None` for every supported one.
    #[getter]
    pub fn maintained_indexes(&self) -> Option<Vec<String>> {
        self.inner.maintained_indexes().map(<[String]>::to_vec)
@@ -403,14 +403,14 @@ impl MemWalWriteSpec {
    }
 }

-impl From<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
-    fn from(spec: MemWalWriteSpec) -> Self {
+impl From<RTEWriteSpec> for lancedb::table::RTEWriteSpec {
+    fn from(spec: RTEWriteSpec) -> Self {
        spec.inner
    }
 }

-impl From<lancedb::table::MemWalWriteSpec> for MemWalWriteSpec {
-    fn from(inner: lancedb::table::MemWalWriteSpec) -> Self {
+impl From<lancedb::table::RTEWriteSpec> for RTEWriteSpec {
+    fn from(inner: lancedb::table::RTEWriteSpec) -> Self {
        Self { inner }
    }
 }
@@ -1636,8 +1636,8 @@ impl Table {
        if let Some(use_index) = parameters.use_index {
            builder.use_index(use_index);
        }
-        if let Some(use_memwal) = parameters.use_memwal {
-            builder.use_memwal(use_memwal);
+        if let Some(use_rte) = parameters.use_rte {
+            builder.use_rte(use_rte);
        }
        if let Some(validate_single_shard) = parameters.validate_single_shard {
            builder.validate_single_shard(validate_single_shard);
@@ -1662,29 +1662,29 @@ impl Table {
        })
    }

-    pub fn enable_memwal<'a>(
+    pub fn enable_rte<'a>(
        self_: PyRef<'a, Self>,
-        spec: MemWalWriteSpec,
+        spec: RTEWriteSpec,
    ) -> PyResult<Bound<'a, PyAny>> {
        let inner = self_.inner_ref()?.clone();
-        let native_spec = lancedb::table::MemWalWriteSpec::from(spec);
+        let native_spec = lancedb::table::RTEWriteSpec::from(spec);
        future_into_py(self_.py(), async move {
-            inner.enable_memwal(native_spec).await.infer_error()
+            inner.enable_rte(native_spec).await.infer_error()
        })
    }

-    pub fn disable_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn disable_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            inner.disable_memwal().await.infer_error()
+            inner.disable_rte().await.infer_error()
        })
    }

-    pub fn get_memwal_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn get_rte_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            let spec = inner.get_memwal_spec().await.infer_error()?;
-            Ok(spec.map(MemWalWriteSpec::from))
+            let spec = inner.get_rte_spec().await.infer_error()?;
+            Ok(spec.map(RTEWriteSpec::from))
        })
    }

@@ -1703,54 +1703,55 @@ impl Table {
        Ok(())
    }

-    /// Converge the table's MemWAL write path into its base table.
+    /// Converge the table's RTE write path into its base table.
    ///
    /// Best-effort: with writes flowing, new rows may land after the last
    /// pass. Errors if the table stops making progress.
-    pub fn checkpoint_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn checkpoint_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            inner.checkpoint_memwal().await.infer_error()
+            inner.checkpoint_rte().await.infer_error()
        })
    }

    /// Freeze every table shard's active memtable into an SSTable.
-    pub fn flush_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn flush_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
-        future_into_py(self_.py(), async move {
-            inner.flush_memwal().await.infer_error()
-        })
+        future_into_py(
+            self_.py(),
+            async move { inner.flush_rte().await.infer_error() },
+        )
    }

    /// Trigger a background SSTable compaction pass per table shard. Returns once the
-    /// passes are dispatched, not once they finish — watch `get_memwal_stats`.
-    pub fn compact_memwal(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    /// passes are dispatched, not once they finish — watch `get_rte_stats`.
+    pub fn compact_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            inner.compact_memwal().await.infer_error()
+            inner.compact_rte().await.infer_error()
        })
    }

-    /// Live MemWAL state, or `None` when the MemWAL write path is not enabled.
+    /// Live RTE state, or `None` when the RTE write path is not enabled.
    #[pyo3(signature = (include_sstable_rows=false))]
-    pub fn get_memwal_stats(
+    pub fn get_rte_stats(
        self_: PyRef<'_, Self>,
        include_sstable_rows: bool,
    ) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
            let stats = inner
-                .get_memwal_stats(include_sstable_rows)
+                .get_rte_stats(include_sstable_rows)
                .await
                .infer_error()?;
-            Python::attach(|py| stats.map(|s| memwal_stats_to_py(py, &s)).transpose())
+            Python::attach(|py| stats.map(|s| rte_stats_to_py(py, &s)).transpose())
        })
    }

-    pub fn close_memwal_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn close_rte_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
        let inner = self_.inner_ref()?.clone();
        future_into_py(self_.py(), async move {
-            inner.close_memwal_writers().await.infer_error()
+            inner.close_rte_writers().await.infer_error()
        })
    }

@@ -2082,7 +2083,7 @@ pub struct MergeInsertParams {
    when_not_matched_by_source_condition_expr: Option<PyExpr>,
    timeout: Option<std::time::Duration>,
    use_index: Option<bool>,
-    use_memwal: Option<bool>,
+    use_rte: Option<bool>,
    validate_single_shard: Option<bool>,
 }