From 00ae2cdf7a426da8b965e0551a9a6e3203f65dca Mon Sep 17 00:00:00 2001
From: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Date: Sun, 25 Oct 2026 10:15:54 +0000
Subject: [PATCH] docs: add Rust compaction tuning example
---
docs/web/maintenance/compaction.mdx | 16 ++++-
docs/web/snippets/doc_storage_examples.mdx | 2 +
rust/lancedb/tests/doc_storage_examples.rs | 76 ++++++++++++++++++++++
3 files changed, 91 insertions(+), 3 deletions(-)
create mode 100644 rust/lancedb/tests/doc_storage_examples.rs
diff --git a/docs/web/maintenance/compaction.mdx b/docs/web/maintenance/compaction.mdx
index 0a75c6b13..5d2a9c42f 100644
--- a/docs/web/maintenance/compaction.mdx
+++ b/docs/web/maintenance/compaction.mdx
@@ -12,7 +12,7 @@ availability:
background, and `optimize()` is not available there.
---
-import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
+import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex, RsStorageCompactionTuning } from '/snippets/doc_storage_examples.mdx';
Every write commits a new table version, and versions accumulate: small files
from frequent appends, superseded files from updates and deletes, and rows added
@@ -111,8 +111,8 @@ until the first supported release is confirmed.
### Tune compaction for wide rows {#tune-wide-rows}
-For an existing local Python table, derive a fragment row target from the
-expected row size. This example uses rows averaging 1 KiB and a 64 KiB fragment
+For an existing local table, derive a fragment row target from the expected
+row size. These examples use rows averaging 1 KiB and a 64 KiB fragment
target; choose sizes for your workload. Smaller scan batches and one compaction
task at a time can also reduce memory use.
@@ -120,8 +120,18 @@ task at a time can also reduce memory use.
{PyStorageCompactionTuning}
+
+ {RsStorageCompactionTuning}
+
+The Rust example runs compaction only. Run pruning and index optimization
+separately when needed; Python's `optimize()` includes those steps.
+
+TypeScript's `optimize()` does not currently expose the fragment row target,
+scan batch size, or compaction thread count. These tuning examples apply to
+Python and Rust local tables.
+
Planning remains row based. The row target bounds later passes after existing
oversized fragments have been rewritten; the first pass rewrites those
fragments whole, so peak memory can exceed the target. `max_bytes_per_file`
diff --git a/docs/web/snippets/doc_storage_examples.mdx b/docs/web/snippets/doc_storage_examples.mdx
index a598428ff..f3b6be6a6 100644
--- a/docs/web/snippets/doc_storage_examples.mdx
+++ b/docs/web/snippets/doc_storage_examples.mdx
@@ -4,3 +4,5 @@ export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\
export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n \"vector\",\n config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n replace=True,\n)\n";
+export const RsStorageCompactionTuning = "use lancedb::table::{CompactionOptions, OptimizeAction};\n\nlet desired_fragment_bytes: usize = 64 * 1024;\nlet average_row_bytes: usize = 1024; // Estimate from representative rows.\nlet options = CompactionOptions {\n target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),\n batch_size: Some(16),\n num_threads: Some(1),\n ..Default::default()\n};\ntable\n .optimize(OptimizeAction::Compact {\n options,\n remap_options: None,\n })\n .await?;\n";
+
diff --git a/rust/lancedb/tests/doc_storage_examples.rs b/rust/lancedb/tests/doc_storage_examples.rs
new file mode 100644
index 000000000..dd3acc597
--- /dev/null
+++ b/rust/lancedb/tests/doc_storage_examples.rs
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::ops::Range;
+use std::sync::Arc;
+
+use arrow_array::{
+ ArrayRef, BinaryArray, Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
+};
+
+fn wide_rows(ids: Range) -> anyhow::Result> {
+ let payload = [b'x'; 1016];
+ let batch = RecordBatch::try_from_iter(vec![
+ (
+ "id",
+ Arc::new(Int64Array::from_iter_values(ids.clone())) as ArrayRef,
+ ),
+ (
+ "payload",
+ Arc::new(BinaryArray::from_iter_values(
+ ids.map(|_| payload.as_slice()),
+ )) as ArrayRef,
+ ),
+ ])?;
+ let schema = batch.schema();
+ Ok(Box::new(RecordBatchIterator::new(
+ std::iter::once(Ok(batch)),
+ schema,
+ )))
+}
+
+async fn tune_compaction(table: &lancedb::Table) -> lancedb::Result<()> {
+ // --8<-- [start:storage_compaction_tuning]
+ use lancedb::table::{CompactionOptions, OptimizeAction};
+
+ let desired_fragment_bytes: usize = 64 * 1024;
+ let average_row_bytes: usize = 1024; // Estimate from representative rows.
+ let options = CompactionOptions {
+ target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),
+ batch_size: Some(16),
+ num_threads: Some(1),
+ ..Default::default()
+ };
+ table
+ .optimize(OptimizeAction::Compact {
+ options,
+ remap_options: None,
+ })
+ .await?;
+ // --8<-- [end:storage_compaction_tuning]
+ Ok(())
+}
+
+#[tokio::test]
+async fn compaction_tuning() -> anyhow::Result<()> {
+ let directory = tempfile::tempdir()?;
+ let db = lancedb::connect(directory.path().to_str().unwrap())
+ .execute()
+ .await?;
+ let table = db
+ .create_table("compaction_tuning", wide_rows(0..32)?)
+ .execute()
+ .await?;
+ table.add(wide_rows(32..64)?).execute().await?;
+ assert_eq!(table.stats().await?.fragment_stats.num_fragments, 2);
+
+ tune_compaction(&table).await?;
+ assert_eq!(table.stats().await?.fragment_stats.num_fragments, 1);
+ assert_eq!(table.count_rows(None).await?, 64);
+
+ table.add(wide_rows(64..65)?).execute().await?;
+ tune_compaction(&table).await?;
+ assert_eq!(table.stats().await?.fragment_stats.num_fragments, 2);
+ assert_eq!(table.count_rows(None).await?, 65);
+ Ok(())
+}