mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-25 13:12:39 +00:00
docs: add Rust compaction tuning example
This commit is contained in:
@@ -12,7 +12,7 @@ availability:
|
||||
background, and `optimize()` is not available there.
|
||||
---
|
||||
|
||||
import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
|
||||
import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex, RsStorageCompactionTuning } from '/snippets/doc_storage_examples.mdx';
|
||||
|
||||
Every write commits a new table version, and versions accumulate: small files
|
||||
from frequent appends, superseded files from updates and deletes, and rows added
|
||||
@@ -111,8 +111,8 @@ until the first supported release is confirmed.
|
||||
|
||||
### Tune compaction for wide rows {#tune-wide-rows}
|
||||
|
||||
For an existing local Python table, derive a fragment row target from the
|
||||
expected row size. This example uses rows averaging 1 KiB and a 64 KiB fragment
|
||||
For an existing local table, derive a fragment row target from the expected
|
||||
row size. These examples use rows averaging 1 KiB and a 64 KiB fragment
|
||||
target; choose sizes for your workload. Smaller scan batches and one compaction
|
||||
task at a time can also reduce memory use.
|
||||
|
||||
@@ -120,8 +120,18 @@ task at a time can also reduce memory use.
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyStorageCompactionTuning}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsStorageCompactionTuning}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
The Rust example runs compaction only. Run pruning and index optimization
|
||||
separately when needed; Python's `optimize()` includes those steps.
|
||||
|
||||
TypeScript's `optimize()` does not currently expose the fragment row target,
|
||||
scan batch size, or compaction thread count. These tuning examples apply to
|
||||
Python and Rust local tables.
|
||||
|
||||
Planning remains row based. The row target bounds later passes after existing
|
||||
oversized fragments have been rewritten; the first pass rewrites those
|
||||
fragments whole, so peak memory can exceed the target. `max_bytes_per_file`
|
||||
|
||||
@@ -4,3 +4,5 @@ export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\
|
||||
|
||||
export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n \"vector\",\n config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n replace=True,\n)\n";
|
||||
|
||||
export const RsStorageCompactionTuning = "use lancedb::table::{CompactionOptions, OptimizeAction};\n\nlet desired_fragment_bytes: usize = 64 * 1024;\nlet average_row_bytes: usize = 1024; // Estimate from representative rows.\nlet options = CompactionOptions {\n target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),\n batch_size: Some(16),\n num_threads: Some(1),\n ..Default::default()\n};\ntable\n .optimize(OptimizeAction::Compact {\n options,\n remap_options: None,\n })\n .await?;\n";
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::ops::Range;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_array::{
|
||||
ArrayRef, BinaryArray, Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader,
|
||||
};
|
||||
|
||||
fn wide_rows(ids: Range<i64>) -> anyhow::Result<Box<dyn RecordBatchReader + Send>> {
|
||||
let payload = [b'x'; 1016];
|
||||
let batch = RecordBatch::try_from_iter(vec![
|
||||
(
|
||||
"id",
|
||||
Arc::new(Int64Array::from_iter_values(ids.clone())) as ArrayRef,
|
||||
),
|
||||
(
|
||||
"payload",
|
||||
Arc::new(BinaryArray::from_iter_values(
|
||||
ids.map(|_| payload.as_slice()),
|
||||
)) as ArrayRef,
|
||||
),
|
||||
])?;
|
||||
let schema = batch.schema();
|
||||
Ok(Box::new(RecordBatchIterator::new(
|
||||
std::iter::once(Ok(batch)),
|
||||
schema,
|
||||
)))
|
||||
}
|
||||
|
||||
async fn tune_compaction(table: &lancedb::Table) -> lancedb::Result<()> {
|
||||
// --8<-- [start:storage_compaction_tuning]
|
||||
use lancedb::table::{CompactionOptions, OptimizeAction};
|
||||
|
||||
let desired_fragment_bytes: usize = 64 * 1024;
|
||||
let average_row_bytes: usize = 1024; // Estimate from representative rows.
|
||||
let options = CompactionOptions {
|
||||
target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),
|
||||
batch_size: Some(16),
|
||||
num_threads: Some(1),
|
||||
..Default::default()
|
||||
};
|
||||
table
|
||||
.optimize(OptimizeAction::Compact {
|
||||
options,
|
||||
remap_options: None,
|
||||
})
|
||||
.await?;
|
||||
// --8<-- [end:storage_compaction_tuning]
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compaction_tuning() -> anyhow::Result<()> {
|
||||
let directory = tempfile::tempdir()?;
|
||||
let db = lancedb::connect(directory.path().to_str().unwrap())
|
||||
.execute()
|
||||
.await?;
|
||||
let table = db
|
||||
.create_table("compaction_tuning", wide_rows(0..32)?)
|
||||
.execute()
|
||||
.await?;
|
||||
table.add(wide_rows(32..64)?).execute().await?;
|
||||
assert_eq!(table.stats().await?.fragment_stats.num_fragments, 2);
|
||||
|
||||
tune_compaction(&table).await?;
|
||||
assert_eq!(table.stats().await?.fragment_stats.num_fragments, 1);
|
||||
assert_eq!(table.count_rows(None).await?, 64);
|
||||
|
||||
table.add(wide_rows(64..65)?).execute().await?;
|
||||
tune_compaction(&table).await?;
|
||||
assert_eq!(table.stats().await?.fragment_stats.num_fragments, 2);
|
||||
assert_eq!(table.count_rows(None).await?, 65);
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user