From 7ba76d15ffbaf9297c268cbe39399f22c765585f Mon Sep 17 00:00:00 2001
From: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:41:34 +0000
Subject: [PATCH] docs: draft tested compaction tuning guidance

Derive a row target from estimated row size, link the options reference, and retain first-pass and source-budget caveats. Keep the section hidden until release availability is confirmed.

Validate the example across initial compaction and a later append; all seven storage documentation tests pass. Guide, draft, and snippet MDX compilation and targeted Ruff checks pass.
---
 .../web-tests/py/test_doc_storage_examples.py | 30 +++++++++++++++++++
 docs/web/maintenance/compaction.mdx           | 30 ++++++++++++++++++-
 docs/web/snippets/doc_storage_examples.mdx    |  2 ++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/docs/web-tests/py/test_doc_storage_examples.py b/docs/web-tests/py/test_doc_storage_examples.py
index 13bedb731..99ab4a514 100644
--- a/docs/web-tests/py/test_doc_storage_examples.py
+++ b/docs/web-tests/py/test_doc_storage_examples.py
@@ -7,6 +7,36 @@ import pyarrow as pa
 import pytest
 
 
+def test_compaction_tuning(mem_db):
+    payload = b"x" * 1016
+    table = mem_db.create_table(
+        "compaction_tuning", [{"id": i, "payload": payload} for i in range(32)]
+    )
+    table.add([{"id": i, "payload": payload} for i in range(32, 64)])
+    assert table.stats()["fragment_stats"]["num_fragments"] == 2
+
+    # --8<-- [start:storage_compaction_tuning]
+    from lancedb import CompactionOptions
+
+    desired_fragment_bytes = 64 * 1024
+    average_row_bytes = 1024  # Estimate from representative rows.
+    options: CompactionOptions = {
+        "target_rows_per_fragment": max(1, desired_fragment_bytes // average_row_bytes),
+        "batch_size": 16,
+        "num_threads": 1,
+    }
+    table.optimize(compaction_options=options)
+    # --8<-- [end:storage_compaction_tuning]
+
+    assert table.stats()["fragment_stats"]["num_fragments"] == 1
+    assert table.count_rows() == 64
+
+    table.add([{"id": 64, "payload": payload}])
+    table.optimize(compaction_options=options)
+    assert table.stats()["fragment_stats"]["num_fragments"] == 2
+    assert sorted(table.to_arrow()["id"].to_pylist()) == list(range(65))
+
+
 def test_rebuild_vector_index(tmp_db):
     rng = np.random.default_rng(7)
     table = tmp_db.create_table(
diff --git a/docs/web/maintenance/compaction.mdx b/docs/web/maintenance/compaction.mdx
index 5bee9c65e..0a75c6b13 100644
--- a/docs/web/maintenance/compaction.mdx
+++ b/docs/web/maintenance/compaction.mdx
@@ -12,7 +12,7 @@ availability:
     background, and `optimize()` is not available there.
 ---
 
-import { PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
+import { PyStorageCompactionTuning, PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx';
 
 Every write commits a new table version, and versions accumulate: small files
 from frequent appends, superseded files from updates and deletes, and rows added
@@ -104,3 +104,31 @@ table.optimize(cleanup_older_than=timedelta(days=1))   # reclaim space sooner
 ```
 
 The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates. See [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed. Update or rebuild the index after large update batches.
+
+{/*
+Draft for the Python compaction_options release. Keep this section unpublished
+until the first supported release is confirmed.
+
+### Tune compaction for wide rows {#tune-wide-rows}
+
+For an existing local Python table, derive a fragment row target from the
+expected row size. This example uses rows averaging 1 KiB and a 64 KiB fragment
+target; choose sizes for your workload. Smaller scan batches and one compaction
+task at a time can also reduce memory use.
+
+<CodeGroup>
+    <CodeBlock filename="Python" language="Python" icon="python">
+    {PyStorageCompactionTuning}
+    </CodeBlock>
+</CodeGroup>
+
+Planning remains row based. The row target bounds later passes after existing
+oversized fragments have been rewritten; the first pass rewrites those
+fragments whole, so peak memory can exceed the target. `max_bytes_per_file`
+limits output file size rather than compaction memory.
+
+The `max_source_rows` and `max_source_bytes` budgets admit whole planned tasks.
+If the first task exceeds a budget, that run performs no compaction. See the
+[CompactionOptions reference](https://lancedb.github.io/lancedb/python/python/#lancedb.table.CompactionOptions)
+for all options and their limits.
+*/}
diff --git a/docs/web/snippets/doc_storage_examples.mdx b/docs/web/snippets/doc_storage_examples.mdx
index 47e2da787..a598428ff 100644
--- a/docs/web/snippets/doc_storage_examples.mdx
+++ b/docs/web/snippets/doc_storage_examples.mdx
@@ -1,4 +1,6 @@
 {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */}
 
+export const PyStorageCompactionTuning = "from lancedb import CompactionOptions\n\ndesired_fragment_bytes = 64 * 1024\naverage_row_bytes = 1024  # Estimate from representative rows.\noptions: CompactionOptions = {\n    \"target_rows_per_fragment\": max(1, desired_fragment_bytes // average_row_bytes),\n    \"batch_size\": 16,\n    \"num_threads\": 1,\n}\ntable.optimize(compaction_options=options)\n";
+
 export const PyStorageRebuildVectorIndex = "from lancedb.index import IvfFlat\n\ntable.create_index(\n    \"vector\",\n    config=IvfFlat(distance_type=\"l2\", num_partitions=4),\n    replace=True,\n)\n";