refactor!: rename MemWAL APIs to RTE (realtime engine) (#4598)

## What

Renames the user-facing MemWAL surface to **RTE** (realtime engine)
across Rust, Python, TypeScript, and Java, including the REST wire
format. This follows the LSM → MemWAL rename in #4075.

| Before | After |
|---|---|
| `MemWalWriteSpec` / `MemWalStats` | `RTEWriteSpec` / `RTEStats` |
| `enable_memwal` / `disable_memwal` / `get_memwal_spec` | `enable_rte`
/ `disable_rte` / `get_rte_spec` |
| `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | `flush_rte`
/ `compact_rte` / `checkpoint_rte` |
| `get_memwal_stats` / `close_memwal_writers` | `get_rte_stats` /
`close_rte_writers` |
| `use_memwal` (query, merge insert) | `use_rte` |
| TS/Java `enableMemWal`, `useMemWal`, ... | `enableRTE`, `useRTE`, ...
(napi `js_name` set explicitly) |
| Java `LanceDbTableMemWal` | `LanceDbTableRTE` |
| `table/memwal_stats.rs` | `table/rte_stats.rs` |

Docs and comments now say "realtime engine (RTE)". Text about Lance's
underlying mechanism keeps "MemWAL" ("Lance's MemWAL", "MemWAL index").

**Left alone:** Lance-owned names (`MemWalIndexDetails`,
`DatasetMemWalExt`, `InitializeMemWal*`, `mem_wal` paths), internal
`Lsm*` names and modules, and the `does not support with_row_id` refusal
text that the hybrid fallback matches on.

## Compatibility

Breaking, for an experimental API. The wire format changes, so this must
roll out together with the matching sophon change (lancedb/sophon#8870):

- Routes: `/enable_memwal/` → `/enable_rte/`, `/disable_memwal/` →
`/disable_rte/`, `/get_memwal_spec/` → `/get_rte_spec/`,
`/get_memwal_stats/` → `/get_rte_stats/`, `/flush_memwal/` →
`/flush_rte/`, `/compact_memwal/` → `/compact_rte/`
- Response keys: `memwal_spec` → `rte_spec`, `memwal_stats` →
`rte_stats`
- Request field: `use_memwal` → `use_rte` on query and merge-insert
bodies

The SQL keyword becomes `ALTER TABLE ... SET RTE` server-side. The
guides under `docs/web` will be updated separately.

## Validation

- Rust: `cargo clippy --features remote --tests --examples` clean;
`cargo test --features remote -p lancedb --lib`: 1533 passed
- Java: `mvn test`: 37 passed (Spotless applied with JDK 11)
- Python: `ruff format` / `ruff check` clean; targeted tests
(`test_rte_spec`, `test_merge_insert_lsm`, `test_remote_db`,
`test_wal_hybrid`, `test_query`, `test_permutation`, `test_table`): 708
passed
- TypeScript: `pnpm build`; `table`, `query` tests: 606 passed; `biome
check` clean; `docs/src/js` regenerated with `pnpm run docs`

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dan Rammer
2026-10-20 11:58:57 -05:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 1772f4ef83
commit 92825f6f13
73 changed files with 1656 additions and 1600 deletions
+20 -20
View File
@@ -1844,10 +1844,10 @@ def lsm_test_table(lsm_handler):
"""A remote table whose LSM routes are served by ``lsm_handler``.
``lsm_handler(request, route)`` is called for ``/v1/table/test/<route>/``
where route is one of flush_memwal, compact_memwal, get_memwal_stats, and is
where route is one of flush_rte, compact_rte, get_rte_stats, and is
responsible for writing the response.
"""
routes = ("flush_memwal", "compact_memwal", "get_memwal_stats")
routes = ("flush_rte", "compact_rte", "get_rte_stats")
def handler(request):
match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path)
@@ -1879,7 +1879,7 @@ def send_json(request, payload, status=200):
request.wfile.write(json.dumps(payload).encode())
def test_get_memwal_stats_sync():
def test_get_rte_stats_sync():
"""The sync wrapper round-trips the server payload into a dict."""
table_shard = {
"shard_id": "b0",
@@ -1904,29 +1904,29 @@ def test_get_memwal_stats_sync():
seen_bodies = []
def lsm_handler(request, route):
assert route == "get_memwal_stats"
assert route == "get_rte_stats"
seen_bodies.append(read_json_body(request))
send_json(request, {"memwal_stats": {"table_shards": [table_shard]}})
send_json(request, {"rte_stats": {"table_shards": [table_shard]}})
with lsm_test_table(lsm_handler) as table:
assert table.get_memwal_stats() == {"table_shards": [table_shard]}
assert table.get_rte_stats() == {"table_shards": [table_shard]}
# Off by default, and forwarded when asked for.
assert seen_bodies == [{"include_sstable_rows": False}]
table.get_memwal_stats(include_sstable_rows=True)
table.get_rte_stats(include_sstable_rows=True)
assert seen_bodies[-1] == {"include_sstable_rows": True}
def test_get_memwal_stats_sync_returns_none_when_memwal_disabled():
"""A null envelope means the MemWAL write path is not enabled, not an error."""
def test_get_rte_stats_sync_returns_none_when_rte_disabled():
"""A null envelope means the RTE write path is not enabled, not an error."""
def lsm_handler(request, route):
send_json(request, {"memwal_stats": None})
send_json(request, {"rte_stats": None})
with lsm_test_table(lsm_handler) as table:
assert table.get_memwal_stats() is None
assert table.get_rte_stats() is None
def test_flush_and_compact_memwal_sync():
def test_flush_and_compact_rte_sync():
"""Both are one-shot POSTs answered 202 with no body."""
called = []
@@ -1936,12 +1936,12 @@ def test_flush_and_compact_memwal_sync():
request.end_headers()
with lsm_test_table(lsm_handler) as table:
assert table.flush_memwal() is None
assert table.compact_memwal() is None
assert called == ["flush_memwal", "compact_memwal"]
assert table.flush_rte() is None
assert table.compact_rte() is None
assert called == ["flush_rte", "compact_rte"]
def test_checkpoint_memwal_sync():
def test_checkpoint_rte_sync():
"""Freeze, read the watermark, and return once no SSTables remain.
The convergence loop itself is covered in Rust; this pins the sync
@@ -1951,17 +1951,17 @@ def test_checkpoint_memwal_sync():
def lsm_handler(request, route):
called.append(route)
if route == "get_memwal_stats":
if route == "get_rte_stats":
# An empty SSTable tier yields no target watermark, so the loop is done
# after the freeze without ever polling compaction.
send_json(request, {"memwal_stats": {"table_shards": []}})
send_json(request, {"rte_stats": {"table_shards": []}})
else:
request.send_response(202)
request.end_headers()
with lsm_test_table(lsm_handler) as table:
assert table.checkpoint_memwal() is None
assert called == ["flush_memwal", "get_memwal_stats"]
assert table.checkpoint_rte() is None
assert called == ["flush_rte", "get_rte_stats"]
@contextlib.contextmanager