mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-27 22:16:14 +00:00
refactor!: rename MemWAL APIs to RTE (realtime engine) (#4598)
## What Renames the user-facing MemWAL surface to **RTE** (realtime engine) across Rust, Python, TypeScript, and Java, including the REST wire format. This follows the LSM → MemWAL rename in #4075. | Before | After | |---|---| | `MemWalWriteSpec` / `MemWalStats` | `RTEWriteSpec` / `RTEStats` | | `enable_memwal` / `disable_memwal` / `get_memwal_spec` | `enable_rte` / `disable_rte` / `get_rte_spec` | | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | `flush_rte` / `compact_rte` / `checkpoint_rte` | | `get_memwal_stats` / `close_memwal_writers` | `get_rte_stats` / `close_rte_writers` | | `use_memwal` (query, merge insert) | `use_rte` | | TS/Java `enableMemWal`, `useMemWal`, ... | `enableRTE`, `useRTE`, ... (napi `js_name` set explicitly) | | Java `LanceDbTableMemWal` | `LanceDbTableRTE` | | `table/memwal_stats.rs` | `table/rte_stats.rs` | Docs and comments now say "realtime engine (RTE)". Text about Lance's underlying mechanism keeps "MemWAL" ("Lance's MemWAL", "MemWAL index"). **Left alone:** Lance-owned names (`MemWalIndexDetails`, `DatasetMemWalExt`, `InitializeMemWal*`, `mem_wal` paths), internal `Lsm*` names and modules, and the `does not support with_row_id` refusal text that the hybrid fallback matches on. ## Compatibility Breaking, for an experimental API. The wire format changes, so this must roll out together with the matching sophon change (lancedb/sophon#8870): - Routes: `/enable_memwal/` → `/enable_rte/`, `/disable_memwal/` → `/disable_rte/`, `/get_memwal_spec/` → `/get_rte_spec/`, `/get_memwal_stats/` → `/get_rte_stats/`, `/flush_memwal/` → `/flush_rte/`, `/compact_memwal/` → `/compact_rte/` - Response keys: `memwal_spec` → `rte_spec`, `memwal_stats` → `rte_stats` - Request field: `use_memwal` → `use_rte` on query and merge-insert bodies The SQL keyword becomes `ALTER TABLE ... SET RTE` server-side. The guides under `docs/web` will be updated separately. ## Validation - Rust: `cargo clippy --features remote --tests --examples` clean; `cargo test --features remote -p lancedb --lib`: 1533 passed - Java: `mvn test`: 37 passed (Spotless applied with JDK 11) - Python: `ruff format` / `ruff check` clean; targeted tests (`test_rte_spec`, `test_merge_insert_lsm`, `test_remote_db`, `test_wal_hybrid`, `test_query`, `test_permutation`, `test_table`): 708 passed - TypeScript: `pnpm build`; `table`, `query` tests: 606 passed; `biome check` clean; `docs/src/js` regenerated with `pnpm run docs` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1772f4ef83
commit
92825f6f13
@@ -1844,10 +1844,10 @@ def lsm_test_table(lsm_handler):
|
||||
"""A remote table whose LSM routes are served by ``lsm_handler``.
|
||||
|
||||
``lsm_handler(request, route)`` is called for ``/v1/table/test/<route>/``
|
||||
where route is one of flush_memwal, compact_memwal, get_memwal_stats, and is
|
||||
where route is one of flush_rte, compact_rte, get_rte_stats, and is
|
||||
responsible for writing the response.
|
||||
"""
|
||||
routes = ("flush_memwal", "compact_memwal", "get_memwal_stats")
|
||||
routes = ("flush_rte", "compact_rte", "get_rte_stats")
|
||||
|
||||
def handler(request):
|
||||
match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path)
|
||||
@@ -1879,7 +1879,7 @@ def send_json(request, payload, status=200):
|
||||
request.wfile.write(json.dumps(payload).encode())
|
||||
|
||||
|
||||
def test_get_memwal_stats_sync():
|
||||
def test_get_rte_stats_sync():
|
||||
"""The sync wrapper round-trips the server payload into a dict."""
|
||||
table_shard = {
|
||||
"shard_id": "b0",
|
||||
@@ -1904,29 +1904,29 @@ def test_get_memwal_stats_sync():
|
||||
seen_bodies = []
|
||||
|
||||
def lsm_handler(request, route):
|
||||
assert route == "get_memwal_stats"
|
||||
assert route == "get_rte_stats"
|
||||
seen_bodies.append(read_json_body(request))
|
||||
send_json(request, {"memwal_stats": {"table_shards": [table_shard]}})
|
||||
send_json(request, {"rte_stats": {"table_shards": [table_shard]}})
|
||||
|
||||
with lsm_test_table(lsm_handler) as table:
|
||||
assert table.get_memwal_stats() == {"table_shards": [table_shard]}
|
||||
assert table.get_rte_stats() == {"table_shards": [table_shard]}
|
||||
# Off by default, and forwarded when asked for.
|
||||
assert seen_bodies == [{"include_sstable_rows": False}]
|
||||
table.get_memwal_stats(include_sstable_rows=True)
|
||||
table.get_rte_stats(include_sstable_rows=True)
|
||||
assert seen_bodies[-1] == {"include_sstable_rows": True}
|
||||
|
||||
|
||||
def test_get_memwal_stats_sync_returns_none_when_memwal_disabled():
|
||||
"""A null envelope means the MemWAL write path is not enabled, not an error."""
|
||||
def test_get_rte_stats_sync_returns_none_when_rte_disabled():
|
||||
"""A null envelope means the RTE write path is not enabled, not an error."""
|
||||
|
||||
def lsm_handler(request, route):
|
||||
send_json(request, {"memwal_stats": None})
|
||||
send_json(request, {"rte_stats": None})
|
||||
|
||||
with lsm_test_table(lsm_handler) as table:
|
||||
assert table.get_memwal_stats() is None
|
||||
assert table.get_rte_stats() is None
|
||||
|
||||
|
||||
def test_flush_and_compact_memwal_sync():
|
||||
def test_flush_and_compact_rte_sync():
|
||||
"""Both are one-shot POSTs answered 202 with no body."""
|
||||
called = []
|
||||
|
||||
@@ -1936,12 +1936,12 @@ def test_flush_and_compact_memwal_sync():
|
||||
request.end_headers()
|
||||
|
||||
with lsm_test_table(lsm_handler) as table:
|
||||
assert table.flush_memwal() is None
|
||||
assert table.compact_memwal() is None
|
||||
assert called == ["flush_memwal", "compact_memwal"]
|
||||
assert table.flush_rte() is None
|
||||
assert table.compact_rte() is None
|
||||
assert called == ["flush_rte", "compact_rte"]
|
||||
|
||||
|
||||
def test_checkpoint_memwal_sync():
|
||||
def test_checkpoint_rte_sync():
|
||||
"""Freeze, read the watermark, and return once no SSTables remain.
|
||||
|
||||
The convergence loop itself is covered in Rust; this pins the sync
|
||||
@@ -1951,17 +1951,17 @@ def test_checkpoint_memwal_sync():
|
||||
|
||||
def lsm_handler(request, route):
|
||||
called.append(route)
|
||||
if route == "get_memwal_stats":
|
||||
if route == "get_rte_stats":
|
||||
# An empty SSTable tier yields no target watermark, so the loop is done
|
||||
# after the freeze without ever polling compaction.
|
||||
send_json(request, {"memwal_stats": {"table_shards": []}})
|
||||
send_json(request, {"rte_stats": {"table_shards": []}})
|
||||
else:
|
||||
request.send_response(202)
|
||||
request.end_headers()
|
||||
|
||||
with lsm_test_table(lsm_handler) as table:
|
||||
assert table.checkpoint_memwal() is None
|
||||
assert called == ["flush_memwal", "get_memwal_stats"]
|
||||
assert table.checkpoint_rte() is None
|
||||
assert called == ["flush_rte", "get_rte_stats"]
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
|
||||
Reference in New Issue
Block a user