fix(python): expose optimize compaction options (#3870)

## Summary
- expose typed compaction options on synchronous and asynchronous table
optimization
- translate validated Python options into native Lance compaction
settings
- document the row-based steady-state limit and first-pass/source-budget
caveats
- carry the Lance beta.15 fragment-exclusion option through the typed
Python API

## Root cause
The native Python optimize binding always constructed default compaction
settings, and the public AsyncTable API offered no way to lower the
fragment row target. Wide-row fragments therefore remained compaction
candidates on every later optimize call, repeatedly rewriting the
accumulated table.

## Limitations
This is an explicit row-based mitigation, not byte-native task sizing or
a hard memory cap. Callers must derive target_rows_per_fragment from
expected row size. Existing oversized fragments are rewritten whole on
the first pass; max_bytes_per_file limits output file size rather than
compaction memory; and source budgets admit whole planned tasks, so a
budget below the first task performs no compaction. Splitting tasks by a
byte budget or returning a planner error requires a Lance-side change.

## Validation
- cargo check --release -p lancedb-python
- cargo clippy --quiet -p lancedb-python
- cargo fmt --all --check
- ruff format --check python tests
- ruff check python tests
- uv run --no-sync pytest python/tests/test_table.py -k
optimize_compaction -q (16 passed)

Fixes #2325

<!-- lance-gatekeeper-fix:v1 agent=9b58eee0c79040b19540e259af0e676c
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Co-authored-by: Xuanwo <github@xuanwo.io>

docs/src/python/python.md

@@ -329,6 +329,8 @@ code that used the former remote-only argument order.
 
 ::: lancedb.table.Table
 
+::: lancedb.table.CompactionOptions
+
 ::: lancedb.table.FragmentStatistics
 
 ::: lancedb.table.FragmentSummaryStats

docs/web-tests/py/test_doc_storage_examples.py

@@ -7,6 +7,36 @@ import pyarrow as pa
 import pytest
 
 
+def test_compaction_tuning(mem_db):
+    payload = b"x" * 1016
+    table = mem_db.create_table(
+        "compaction_tuning", [{"id": i, "payload": payload} for i in range(32)]
+    )
+    table.add([{"id": i, "payload": payload} for i in range(32, 64)])
+    assert table.stats()["fragment_stats"]["num_fragments"] == 2
+
+    # --8<-- [start:storage_compaction_tuning]
+    from lancedb import CompactionOptions
+
+    desired_fragment_bytes = 64 * 1024
+    average_row_bytes = 1024  # Estimate from representative rows.
+    options: CompactionOptions = {
+        "target_rows_per_fragment": max(1, desired_fragment_bytes // average_row_bytes),
+        "batch_size": 16,
+        "num_threads": 1,
+    }
+    table.optimize(compaction_options=options)
+    # --8<-- [end:storage_compaction_tuning]
+
+    assert table.stats()["fragment_stats"]["num_fragments"] == 1
+    assert table.count_rows() == 64
+
+    table.add([{"id": 64, "payload": payload}])
+    table.optimize(compaction_options=options)
+    assert table.stats()["fragment_stats"]["num_fragments"] == 2
+    assert sorted(table.to_arrow()["id"].to_pylist()) == list(range(65))
+
+
 def test_rebuild_vector_index(tmp_db):
     rng = np.random.default_rng(7)
     table = tmp_db.create_table(

docs/web/maintenance/compaction.mdx

@@ -12,7 +12,7 @@ availability:
     background, and `optimize()` is not available there.
 ---
 
-import { PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
+import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex, RsStorageCompactionTuning } from '/snippets/doc_storage_examples.mdx';
 
 Every write commits a new table version, and versions accumulate: small files
 from frequent appends, superseded files from updates and deletes, and rows added
@@ -104,3 +104,41 @@ table.optimize(cleanup_older_than=timedelta(days=1))   # reclaim space sooner
 ```
 
 The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates. See [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed. Update or rebuild the index after large update batches.
+
+{/*
+Draft for the Python compaction_options release. Keep this section unpublished
+until the first supported release is confirmed.
+
+### Tune compaction for wide rows {#tune-wide-rows}
+
+For an existing local table, derive a fragment row target from the expected
+row size. These examples use rows averaging 1 KiB and a 64 KiB fragment
+target; choose sizes for your workload. Smaller scan batches and one compaction
+task at a time can also reduce memory use.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {PyStorageCompactionTuning}
+    </CodeBlock>
+    <CodeBlock filename="Rust" language="Rust" icon="rust">
+    {RsStorageCompactionTuning}
+    </CodeBlock>
+</CodeGroup>
+
+The Rust example runs compaction only. Run pruning and index optimization
+separately when needed; Python's `optimize()` includes those steps.
+
+TypeScript's `optimize()` does not currently expose the fragment row target,
+scan batch size, or compaction thread count. These tuning examples apply to
+Python and Rust local tables.
+
+Planning remains row based. The row target bounds later passes after existing
+oversized fragments have been rewritten; the first pass rewrites those
+fragments whole, so peak memory can exceed the target. `max_bytes_per_file`
+limits output file size rather than compaction memory.
+
+The `max_source_rows` and `max_source_bytes` budgets admit whole planned tasks.
+If the first task exceeds a budget, that run performs no compaction. See the
+[CompactionOptions reference](https://lancedb.github.io/lancedb/python/python/#lancedb.table.CompactionOptions)
+for all options and their limits.
+*/}

docs/web/snippets/doc_storage_examples.mdx

@@ -1,4 +1,8 @@
 {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
 
+export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\n\ndesired_fragment_bytes = 64 * 1024\naverage_row_bytes = 1024  # Estimate from representative rows.\noptions: CompactionOptions = {\n    \"target_rows_per_fragment\": max(1, desired_fragment_bytes // average_row_bytes),\n    \"batch_size\": 16,\n    \"num_threads\": 1,\n}\ntable.optimize(compaction_options=options)\n";
+
 export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n    \"vector\",\n    config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n    replace=True,\n)\n";
 
+export const RsStorageCompactionTuning = "use lancedb::table::{CompactionOptions, OptimizeAction};\n\nlet desired_fragment_bytes: usize = 64 * 1024;\nlet average_row_bytes: usize = 1024; // Estimate from representative rows.\nlet options = CompactionOptions {\n    target_rows_per_fragment: (desired_fragment_bytes / average_row_bytes).max(1),\n    batch_size: Some(16),\n    num_threads: Some(1),\n    ..Default::default()\n};\ntable\n    .optimize(OptimizeAction::Compact {\n        options,\n        remap_options: None,\n    })\n    .await?;\n";
+