docs: add Rust compaction tuning example

This commit is contained in:
Gatefixer
2026-10-25 10:15:54 +00:00
parent 10541cc2d8
commit 00ae2cdf7a
3 changed files with 91 additions and 3 deletions
+13 -3
View File
@@ -12,7 +12,7 @@ availability:
background, and `optimize()` is not available there.
---
import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex, RsStorageCompactionTuning } from '/snippets/doc_storage_examples.mdx';
Every write commits a new table version, and versions accumulate: small files
from frequent appends, superseded files from updates and deletes, and rows added
@@ -111,8 +111,8 @@ until the first supported release is confirmed.
### Tune compaction for wide rows {#tune-wide-rows}
For an existing local Python table, derive a fragment row target from the
expected row size. This example uses rows averaging 1 KiB and a 64 KiB fragment
For an existing local table, derive a fragment row target from the expected
row size. These examples use rows averaging 1 KiB and a 64 KiB fragment
target; choose sizes for your workload. Smaller scan batches and one compaction
task at a time can also reduce memory use.
@@ -120,8 +120,18 @@ task at a time can also reduce memory use.
<CodeBlock filename="Python" language="Python" icon="python">
{PyStorageCompactionTuning}
</CodeBlock>
<CodeBlock filename="Rust" language="Rust" icon="rust">
{RsStorageCompactionTuning}
</CodeBlock>
</CodeGroup>
The Rust example runs compaction only. Run pruning and index optimization
separately when needed; Python's `optimize()` includes those steps.
TypeScript's `optimize()` does not currently expose the fragment row target,
scan batch size, or compaction thread count. These tuning examples apply to
Python and Rust local tables.
Planning remains row based. The row target bounds later passes after existing
oversized fragments have been rewritten; the first pass rewrites those
fragments whole, so peak memory can exceed the target. `max_bytes_per_file`
@@ -4,3 +4,5 @@ export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\
export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n \"vector\",\n config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n replace=True,\n)\n";
export const RsStorageCompactionTuning = "use lancedb::table::{CompactionOptions, OptimizeAction};\n\nlet desired_fragment_bytes: usize = 64 * 1024;\nlet average_row_bytes: usize = 1024; // Estimate from representative rows.\nlet options = CompactionOptions {\n target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),\n batch_size: Some(16),\n num_threads: Some(1),\n ..Default::default()\n};\ntable\n .optimize(OptimizeAction::Compact {\n options,\n remap_options: None,\n })\n .await?;\n";
@@ -0,0 +1,76 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::ops::Range;
use std::sync::Arc;
use arrow_array::{
ArrayRef, BinaryArray, Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
};
fn wide_rows(ids: Range<i64>) -> anyhow::Result<Box<dyn RecordBatchReader + Send>> {
let payload = [b'x'; 1016];
let batch = RecordBatch::try_from_iter(vec![
(
"id",
Arc::new(Int64Array::from_iter_values(ids.clone())) as ArrayRef,
),
(
"payload",
Arc::new(BinaryArray::from_iter_values(
ids.map(|_| payload.as_slice()),
)) as ArrayRef,
),
])?;
let schema = batch.schema();
Ok(Box::new(RecordBatchIterator::new(
std::iter::once(Ok(batch)),
schema,
)))
}
async fn tune_compaction(table: &lancedb::Table) -> lancedb::Result<()> {
// --8<-- [start:storage_compaction_tuning]
use lancedb::table::{CompactionOptions, OptimizeAction};
let desired_fragment_bytes: usize = 64 * 1024;
let average_row_bytes: usize = 1024; // Estimate from representative rows.
let options = CompactionOptions {
target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),
batch_size: Some(16),
num_threads: Some(1),
..Default::default()
};
table
.optimize(OptimizeAction::Compact {
options,
remap_options: None,
})
.await?;
// --8<-- [end:storage_compaction_tuning]
Ok(())
}
#[tokio::test]
async fn compaction_tuning() -> anyhow::Result<()> {
let directory = tempfile::tempdir()?;
let db = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await?;
let table = db
.create_table("compaction_tuning", wide_rows(0..32)?)
.execute()
.await?;
table.add(wide_rows(32..64)?).execute().await?;
assert_eq!(table.stats().await?.fragment_stats.num_fragments, 2);
tune_compaction(&table).await?;
assert_eq!(table.stats().await?.fragment_stats.num_fragments, 1);
assert_eq!(table.count_rows(None).await?, 64);
table.add(wide_rows(64..65)?).execute().await?;
tune_compaction(&table).await?;
assert_eq!(table.stats().await?.fragment_stats.num_fragments, 2);
assert_eq!(table.count_rows(None).await?, 65);
Ok(())
}