mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-26 21:46:15 +00:00
docs: add Rust compaction tuning example
This commit is contained in:
@@ -12,7 +12,7 @@ availability:
|
||||
background, and `optimize()` is not available there.
|
||||
---
|
||||
|
||||
import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
|
||||
import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex, RsStorageCompactionTuning } from '/snippets/doc_storage_examples.mdx';
|
||||
|
||||
Every write commits a new table version, and versions accumulate: small files
|
||||
from frequent appends, superseded files from updates and deletes, and rows added
|
||||
@@ -111,8 +111,8 @@ until the first supported release is confirmed.
|
||||
|
||||
### Tune compaction for wide rows {#tune-wide-rows}
|
||||
|
||||
For an existing local Python table, derive a fragment row target from the
|
||||
expected row size. This example uses rows averaging 1 KiB and a 64 KiB fragment
|
||||
For an existing local table, derive a fragment row target from the expected
|
||||
row size. These examples use rows averaging 1 KiB and a 64 KiB fragment
|
||||
target; choose sizes for your workload. Smaller scan batches and one compaction
|
||||
task at a time can also reduce memory use.
|
||||
|
||||
@@ -120,8 +120,18 @@ task at a time can also reduce memory use.
|
||||
<CodeBlock filename="Python" language="Python" icon="python">
|
||||
{PyStorageCompactionTuning}
|
||||
</CodeBlock>
|
||||
<CodeBlock filename="Rust" language="Rust" icon="rust">
|
||||
{RsStorageCompactionTuning}
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
The Rust example runs compaction only. Run pruning and index optimization
|
||||
separately when needed; Python's `optimize()` includes those steps.
|
||||
|
||||
TypeScript's `optimize()` does not currently expose the fragment row target,
|
||||
scan batch size, or compaction thread count. These tuning examples apply to
|
||||
Python and Rust local tables.
|
||||
|
||||
Planning remains row based. The row target bounds later passes after existing
|
||||
oversized fragments have been rewritten; the first pass rewrites those
|
||||
fragments whole, so peak memory can exceed the target. `max_bytes_per_file`
|
||||
|
||||
@@ -4,3 +4,5 @@ export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\
|
||||
|
||||
export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n \"vector\",\n config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n replace=True,\n)\n";
|
||||
|
||||
export const RsStorageCompactionTuning = "use lancedb::table::{CompactionOptions, OptimizeAction};\n\nlet desired_fragment_bytes: usize = 64 * 1024;\nlet average_row_bytes: usize = 1024; // Estimate from representative rows.\nlet options = CompactionOptions {\n target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),\n batch_size: Some(16),\n num_threads: Some(1),\n ..Default::default()\n};\ntable\n .optimize(OptimizeAction::Compact {\n options,\n remap_options: None,\n })\n .await?;\n";
|
||||
|
||||
|
||||
Reference in New Issue
Block a user