mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-22 19:52:32 +00:00
refactor!: rename MemWAL APIs to RTE (realtime engine) (#4598)
## What Renames the user-facing MemWAL surface to **RTE** (realtime engine) across Rust, Python, TypeScript, and Java, including the REST wire format. This follows the LSM → MemWAL rename in #4075. | Before | After | |---|---| | `MemWalWriteSpec` / `MemWalStats` | `RTEWriteSpec` / `RTEStats` | | `enable_memwal` / `disable_memwal` / `get_memwal_spec` | `enable_rte` / `disable_rte` / `get_rte_spec` | | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` | `flush_rte` / `compact_rte` / `checkpoint_rte` | | `get_memwal_stats` / `close_memwal_writers` | `get_rte_stats` / `close_rte_writers` | | `use_memwal` (query, merge insert) | `use_rte` | | TS/Java `enableMemWal`, `useMemWal`, ... | `enableRTE`, `useRTE`, ... (napi `js_name` set explicitly) | | Java `LanceDbTableMemWal` | `LanceDbTableRTE` | | `table/memwal_stats.rs` | `table/rte_stats.rs` | Docs and comments now say "realtime engine (RTE)". Text about Lance's underlying mechanism keeps "MemWAL" ("Lance's MemWAL", "MemWAL index"). **Left alone:** Lance-owned names (`MemWalIndexDetails`, `DatasetMemWalExt`, `InitializeMemWal*`, `mem_wal` paths), internal `Lsm*` names and modules, and the `does not support with_row_id` refusal text that the hybrid fallback matches on. ## Compatibility Breaking, for an experimental API. The wire format changes, so this must roll out together with the matching sophon change (lancedb/sophon#8870): - Routes: `/enable_memwal/` → `/enable_rte/`, `/disable_memwal/` → `/disable_rte/`, `/get_memwal_spec/` → `/get_rte_spec/`, `/get_memwal_stats/` → `/get_rte_stats/`, `/flush_memwal/` → `/flush_rte/`, `/compact_memwal/` → `/compact_rte/` - Response keys: `memwal_spec` → `rte_spec`, `memwal_stats` → `rte_stats` - Request field: `use_memwal` → `use_rte` on query and merge-insert bodies The SQL keyword becomes `ALTER TABLE ... SET RTE` server-side. The guides under `docs/web` will be updated separately. ## Validation - Rust: `cargo clippy --features remote --tests --examples` clean; `cargo test --features remote -p lancedb --lib`: 1533 passed - Java: `mvn test`: 37 passed (Spotless applied with JDK 11) - Python: `ruff format` / `ruff check` clean; targeted tests (`test_rte_spec`, `test_merge_insert_lsm`, `test_remote_db`, `test_wal_hybrid`, `test_query`, `test_permutation`, `test_table`): 708 passed - TypeScript: `pnpm build`; `table`, `query` tests: 606 passed; `biome check` clean; `docs/src/js` regenerated with `pnpm run docs` 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dan Rammer
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
1772f4ef83
commit
92825f6f13
+3
-3
@@ -50,10 +50,10 @@ impl NativeMergeInsertBuilder {
|
||||
this
|
||||
}
|
||||
|
||||
#[napi(js_name = "useMemWal")]
|
||||
pub fn use_memwal(&self, enable: bool) -> Self {
|
||||
#[napi(js_name = "useRTE")]
|
||||
pub fn use_rte(&self, enable: bool) -> Self {
|
||||
let mut this = self.clone();
|
||||
this.inner.use_memwal(enable);
|
||||
this.inner.use_rte(enable);
|
||||
this
|
||||
}
|
||||
|
||||
|
||||
+9
-9
@@ -168,9 +168,9 @@ impl Query {
|
||||
self.inner = self.inner.clone().with_row_id();
|
||||
}
|
||||
|
||||
#[napi(js_name = "useMemWal")]
|
||||
pub fn use_memwal(&mut self, enable: bool) {
|
||||
self.inner = self.inner.clone().use_memwal(enable);
|
||||
#[napi(js_name = "useRTE")]
|
||||
pub fn use_rte(&mut self, enable: bool) {
|
||||
self.inner = self.inner.clone().use_rte(enable);
|
||||
}
|
||||
|
||||
#[napi]
|
||||
@@ -399,9 +399,9 @@ impl VectorQuery {
|
||||
self.inner = self.inner.clone().with_row_id();
|
||||
}
|
||||
|
||||
#[napi(js_name = "useMemWal")]
|
||||
pub fn use_memwal(&mut self, enable: bool) {
|
||||
self.inner = self.inner.clone().use_memwal(enable);
|
||||
#[napi(js_name = "useRTE")]
|
||||
pub fn use_rte(&mut self, enable: bool) {
|
||||
self.inner = self.inner.clone().use_rte(enable);
|
||||
}
|
||||
|
||||
#[napi]
|
||||
@@ -514,9 +514,9 @@ impl TakeQuery {
|
||||
self.inner = self.inner.clone().with_row_id();
|
||||
}
|
||||
|
||||
#[napi(js_name = "useMemWal")]
|
||||
pub fn use_memwal(&mut self, enable: bool) {
|
||||
self.inner = self.inner.clone().use_memwal(enable);
|
||||
#[napi(js_name = "useRTE")]
|
||||
pub fn use_rte(&mut self, enable: bool) {
|
||||
self.inner = self.inner.clone().use_rte(enable);
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
|
||||
+47
-50
@@ -580,60 +580,57 @@ impl Table {
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "enableMemWal")]
|
||||
pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> {
|
||||
let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?;
|
||||
#[napi(catch_unwind, js_name = "enableRTE")]
|
||||
pub async fn enable_rte(&self, spec: RTEWriteSpec) -> napi::Result<()> {
|
||||
let native_spec = lancedb::table::RTEWriteSpec::try_from(spec)?;
|
||||
self.inner_ref()?
|
||||
.enable_memwal(native_spec)
|
||||
.enable_rte(native_spec)
|
||||
.await
|
||||
.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "disableMemWal")]
|
||||
pub async fn disable_memwal(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.disable_memwal().await.default_error()
|
||||
#[napi(catch_unwind, js_name = "disableRTE")]
|
||||
pub async fn disable_rte(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.disable_rte().await.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "getMemWalSpec")]
|
||||
pub async fn get_memwal_spec(&self) -> napi::Result<Option<MemWalWriteSpec>> {
|
||||
let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?;
|
||||
Ok(spec.map(MemWalWriteSpec::from))
|
||||
#[napi(catch_unwind, js_name = "getRTESpec")]
|
||||
pub async fn get_rte_spec(&self) -> napi::Result<Option<RTEWriteSpec>> {
|
||||
let spec = self.inner_ref()?.get_rte_spec().await.default_error()?;
|
||||
Ok(spec.map(RTEWriteSpec::from))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "closeMemWalWriters")]
|
||||
pub async fn close_memwal_writers(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?
|
||||
.close_memwal_writers()
|
||||
.await
|
||||
.default_error()
|
||||
#[napi(catch_unwind, js_name = "closeRTEWriters")]
|
||||
pub async fn close_rte_writers(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.close_rte_writers().await.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "flushMemWal")]
|
||||
pub async fn flush_memwal(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.flush_memwal().await.default_error()
|
||||
#[napi(catch_unwind, js_name = "flushRTE")]
|
||||
pub async fn flush_rte(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.flush_rte().await.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "compactMemWal")]
|
||||
pub async fn compact_memwal(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.compact_memwal().await.default_error()
|
||||
#[napi(catch_unwind, js_name = "compactRTE")]
|
||||
pub async fn compact_rte(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.compact_rte().await.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "checkpointMemWal")]
|
||||
pub async fn checkpoint_memwal(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.checkpoint_memwal().await.default_error()
|
||||
#[napi(catch_unwind, js_name = "checkpointRTE")]
|
||||
pub async fn checkpoint_rte(&self) -> napi::Result<()> {
|
||||
self.inner_ref()?.checkpoint_rte().await.default_error()
|
||||
}
|
||||
|
||||
#[napi(catch_unwind, js_name = "getMemWalStats")]
|
||||
pub async fn get_memwal_stats(
|
||||
#[napi(catch_unwind, js_name = "getRTEStats")]
|
||||
pub async fn get_rte_stats(
|
||||
&self,
|
||||
include_sstable_rows: bool,
|
||||
) -> napi::Result<Option<MemWalStats>> {
|
||||
) -> napi::Result<Option<RTEStats>> {
|
||||
let stats = self
|
||||
.inner_ref()?
|
||||
.get_memwal_stats(include_sstable_rows)
|
||||
.get_rte_stats(include_sstable_rows)
|
||||
.await
|
||||
.default_error()?;
|
||||
Ok(stats.map(MemWalStats::from))
|
||||
Ok(stats.map(RTEStats::from))
|
||||
}
|
||||
|
||||
#[napi(catch_unwind)]
|
||||
@@ -946,53 +943,53 @@ impl From<LanceDbFtsToken> for FtsToken {
|
||||
}
|
||||
}
|
||||
|
||||
/// Specification selecting Lance's MemWAL LSM-style write path for
|
||||
/// `mergeInsert`.
|
||||
/// Specification enabling the realtime engine (RTE), which routes
|
||||
/// `mergeInsert` through Lance's MemWAL LSM-style write path.
|
||||
///
|
||||
/// `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
|
||||
/// `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
|
||||
/// `column` is required.
|
||||
#[napi(object)]
|
||||
#[napi(object, js_name = "RTEWriteSpec")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MemWalWriteSpec {
|
||||
pub struct RTEWriteSpec {
|
||||
/// One of `"bucket"`, `"identity"`, or `"unsharded"`.
|
||||
pub spec_type: String,
|
||||
/// Bucket and identity variants: the sharding column.
|
||||
pub column: Option<String>,
|
||||
/// Bucket variant: the number of buckets, in `[1, 1024]`.
|
||||
pub num_buckets: Option<u32>,
|
||||
/// Indexes the MemWAL keeps up to date. Omitted resolves every
|
||||
/// Indexes the RTE keeps up to date. Omitted resolves every
|
||||
/// maintainable index on install; an empty array means none.
|
||||
pub maintained_indexes: Option<Vec<String>>,
|
||||
/// Default `ShardWriter` configuration recorded in the MemWAL index.
|
||||
pub writer_config_defaults: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl TryFrom<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
|
||||
impl TryFrom<RTEWriteSpec> for lancedb::table::RTEWriteSpec {
|
||||
type Error = napi::Error;
|
||||
|
||||
fn try_from(value: MemWalWriteSpec) -> napi::Result<Self> {
|
||||
fn try_from(value: RTEWriteSpec) -> napi::Result<Self> {
|
||||
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
|
||||
let spec = match value.spec_type.as_str() {
|
||||
"bucket" => {
|
||||
let column = value.column.ok_or_else(|| {
|
||||
napi::Error::from_reason("MemWalWriteSpec bucket requires `column`")
|
||||
napi::Error::from_reason("RTEWriteSpec bucket requires `column`")
|
||||
})?;
|
||||
let num_buckets = value.num_buckets.ok_or_else(|| {
|
||||
napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`")
|
||||
napi::Error::from_reason("RTEWriteSpec bucket requires `numBuckets`")
|
||||
})?;
|
||||
Self::bucket(column, num_buckets)
|
||||
}
|
||||
"identity" => {
|
||||
let column = value.column.ok_or_else(|| {
|
||||
napi::Error::from_reason("MemWalWriteSpec identity requires `column`")
|
||||
napi::Error::from_reason("RTEWriteSpec identity requires `column`")
|
||||
})?;
|
||||
Self::identity(column)
|
||||
}
|
||||
"unsharded" => Self::unsharded(),
|
||||
other => {
|
||||
return Err(napi::Error::from_reason(format!(
|
||||
"MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
|
||||
"RTEWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
|
||||
other
|
||||
)));
|
||||
}
|
||||
@@ -1003,9 +1000,9 @@ impl TryFrom<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<lancedb::table::MemWalWriteSpec> for MemWalWriteSpec {
|
||||
fn from(spec: lancedb::table::MemWalWriteSpec) -> Self {
|
||||
use lancedb::table::MemWalWriteSpec as Native;
|
||||
impl From<lancedb::table::RTEWriteSpec> for RTEWriteSpec {
|
||||
fn from(spec: lancedb::table::RTEWriteSpec) -> Self {
|
||||
use lancedb::table::RTEWriteSpec as Native;
|
||||
match spec {
|
||||
Native::Bucket {
|
||||
column,
|
||||
@@ -1148,19 +1145,19 @@ impl From<lancedb::table::TableShardStats> for TableShardStats {
|
||||
}
|
||||
}
|
||||
|
||||
/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
|
||||
/// Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
|
||||
///
|
||||
/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
|
||||
/// the caller's to compute.
|
||||
#[napi(object)]
|
||||
#[napi(object, js_name = "RTEStats")]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MemWalStats {
|
||||
pub struct RTEStats {
|
||||
/// One entry per table shard backing this table.
|
||||
pub table_shards: Vec<TableShardStats>,
|
||||
}
|
||||
|
||||
impl From<lancedb::table::MemWalStats> for MemWalStats {
|
||||
fn from(stats: lancedb::table::MemWalStats) -> Self {
|
||||
impl From<lancedb::table::RTEStats> for RTEStats {
|
||||
fn from(stats: lancedb::table::RTEStats) -> Self {
|
||||
Self {
|
||||
table_shards: stats.table_shards.into_iter().map(Into::into).collect(),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user