mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-28 14:36:13 +00:00
lancedb-gatefixer[bot]andXuanwo
3352bf49df
## Summary - expose typed compaction options on synchronous and asynchronous table optimization - translate validated Python options into native Lance compaction settings - document the row-based steady-state limit and first-pass/source-budget caveats - carry the Lance beta.15 fragment-exclusion option through the typed Python API ## Root cause The native Python optimize binding always constructed default compaction settings, and the public AsyncTable API offered no way to lower the fragment row target. Wide-row fragments therefore remained compaction candidates on every later optimize call, repeatedly rewriting the accumulated table. ## Limitations This is an explicit row-based mitigation, not byte-native task sizing or a hard memory cap. Callers must derive target_rows_per_fragment from expected row size. Existing oversized fragments are rewritten whole on the first pass; max_bytes_per_file limits output file size rather than compaction memory; and source budgets admit whole planned tasks, so a budget below the first task performs no compaction. Splitting tasks by a byte budget or returning a planner error requires a Lance-side change. ## Validation - cargo check --release -p lancedb-python - cargo clippy --quiet -p lancedb-python - cargo fmt --all --check - ruff format --check python tests - ruff check python tests - uv run --no-sync pytest python/tests/test_table.py -k optimize_compaction -q (16 passed) Fixes #2325 <!-- lance-gatekeeper-fix:v1 agent=9b58eee0c79040b19540e259af0e676c generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo <github@xuanwo.io>
9 lines
1.2 KiB
Plaintext
9 lines
1.2 KiB
Plaintext
{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
|
|
|
|
export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\n\ndesired_fragment_bytes = 64 * 1024\naverage_row_bytes = 1024 # Estimate from representative rows.\noptions: CompactionOptions = {\n \"target_rows_per_fragment\": max(1, desired_fragment_bytes // average_row_bytes),\n \"batch_size\": 16,\n \"num_threads\": 1,\n}\ntable.optimize(compaction_options=options)\n";
|
|
|
|
export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n \"vector\",\n config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n replace=True,\n)\n";
|
|
|
|
export const RsStorageCompactionTuning = "use lancedb::table::{CompactionOptions, OptimizeAction};\n\nlet desired_fragment_bytes: usize = 64 * 1024;\nlet average_row_bytes: usize = 1024; // Estimate from representative rows.\nlet options = CompactionOptions {\n target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),\n batch_size: Some(16),\n num_threads: Some(1),\n ..Default::default()\n};\ntable\n .optimize(OptimizeAction::Compact {\n options,\n remap_options: None,\n })\n .await?;\n";
|
|
|