mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
feat: support remote table compaction via SQL OPTIMIZE TABLE statement (#4802)
`Table::optimize()` previously returned `NotSupported` for remote tables. This PR wires it to the LanceDB Enterprise server's `OPTIMIZE TABLE ... REWRITE DATA` SQL statement, submitted over Flight SQL. Only `OptimizeAction::Compact` without a custom `remap_options` or non-default `CompactionOptions` is supported remotely. `OptimizeAction::All`, `::Prune`, and `::Index` all return a clear `NotSupported` error instead of silently doing the wrong thing. A branch-scoped table handle submits a `WITH (branch = '<branch>')` clause; the statement doesn't validate the branch exists itself, so an unknown branch surfaces as an async job failure rather than an error from this call. The server's SQL job adapter doesn't expose compaction metrics, so `OptimizeStats::compaction` is `Some(CompactionMetrics::default())`, a placeholder rather than real numbers — it must still be `Some`, since callers (including the Python/Node bindings) unwrap it unconditionally on success. This mirrors how `refresh_materialized_view_async` already handles the same limitation for its own metrics. The Python and Node `optimize()` convenience methods call a new shared `Table::optimize_full`, which runs `Compact` and then, on a remote table, skips `Prune` and `Index` instead of attempting and failing on them — returning zeroed prune stats rather than submitting a compaction job and then raising. Python's `RemoteTable.optimize()` also now forwards `compaction_options` to the Rust layer, so a non-default value is rejected before submission instead of being silently dropped. --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
dcda5224ee
commit
a9ad499781
+10
-30
@@ -9,7 +9,7 @@ use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
|
||||
use lancedb::table::{
|
||||
AddDataMode, ColumnAlteration as LanceColumnAlteration,
|
||||
FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken,
|
||||
NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable,
|
||||
NewColumnTransform, Ref, Table as LanceDbTable,
|
||||
};
|
||||
use napi::Env;
|
||||
use napi::bindgen_prelude::*;
|
||||
@@ -735,37 +735,17 @@ impl Table {
|
||||
})
|
||||
.transpose()?;
|
||||
|
||||
let compaction_stats = inner
|
||||
.optimize(OptimizeAction::Compact {
|
||||
options: lancedb::table::CompactionOptions::default(),
|
||||
remap_options: None,
|
||||
})
|
||||
.await
|
||||
.default_error()?
|
||||
.compaction
|
||||
.unwrap();
|
||||
let prune_stats = if let Some(before_timestamp) = before_timestamp {
|
||||
inner
|
||||
.optimize_prune_before(before_timestamp, delete_unverified, None)
|
||||
.await
|
||||
} else {
|
||||
inner
|
||||
.optimize(OptimizeAction::Prune {
|
||||
older_than: None,
|
||||
delete_unverified,
|
||||
error_if_tagged_old_versions: None,
|
||||
})
|
||||
.await
|
||||
}
|
||||
.default_error()?
|
||||
.prune
|
||||
.unwrap();
|
||||
inner
|
||||
.optimize(lancedb::table::OptimizeAction::Index(
|
||||
OptimizeOptions::default(),
|
||||
))
|
||||
let stats = inner
|
||||
.optimize_full(
|
||||
lancedb::table::CompactionOptions::default(),
|
||||
None,
|
||||
before_timestamp,
|
||||
delete_unverified,
|
||||
)
|
||||
.await
|
||||
.default_error()?;
|
||||
let compaction_stats = stats.compaction.unwrap();
|
||||
let prune_stats = stats.prune.unwrap();
|
||||
Ok(OptimizeStats {
|
||||
compaction: CompactionStats {
|
||||
files_added: compaction_stats.files_added as i64,
|
||||
|
||||
Reference in New Issue
Block a user