mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
fix(python): expose optimize compaction options (#3870)
## Summary - expose typed compaction options on synchronous and asynchronous table optimization - translate validated Python options into native Lance compaction settings - document the row-based steady-state limit and first-pass/source-budget caveats - carry the Lance beta.15 fragment-exclusion option through the typed Python API ## Root cause The native Python optimize binding always constructed default compaction settings, and the public AsyncTable API offered no way to lower the fragment row target. Wide-row fragments therefore remained compaction candidates on every later optimize call, repeatedly rewriting the accumulated table. ## Limitations This is an explicit row-based mitigation, not byte-native task sizing or a hard memory cap. Callers must derive target_rows_per_fragment from expected row size. Existing oversized fragments are rewritten whole on the first pass; max_bytes_per_file limits output file size rather than compaction memory; and source budgets admit whole planned tasks, so a budget below the first task performs no compaction. Splitting tasks by a byte budget or returning a planner error requires a Lance-side change. ## Validation - cargo check --release -p lancedb-python - cargo clippy --quiet -p lancedb-python - cargo fmt --all --check - ruff format --check python tests - ruff check python tests - uv run --no-sync pytest python/tests/test_table.py -k optimize_compaction -q (16 passed) Fixes #2325 <!-- lance-gatekeeper-fix:v1 agent=9b58eee0c79040b19540e259af0e676c generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
lancedb-gatefixer[bot]
co-authored by
Xuanwo
Xuanwo
parent
311ded06d0
commit
3352bf49df
@@ -14,7 +14,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal, localcontext
|
||||
from time import sleep
|
||||
from typing import List
|
||||
from typing import Any, List
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import lancedb
|
||||
@@ -5014,6 +5014,134 @@ async def test_optimize(mem_db_async: AsyncConnection):
|
||||
assert await table.query().to_arrow() == pa.table({"x": [[1], [2]]})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_compaction_options(mem_db_async: AsyncConnection):
|
||||
table = await mem_db_async.create_table("test", data=[{"x": 1}])
|
||||
await table.add([{"x": 2}])
|
||||
|
||||
stats = await table.optimize(
|
||||
compaction_options={
|
||||
"target_rows_per_fragment": 1,
|
||||
"batch_size": 1,
|
||||
"num_threads": 1,
|
||||
}
|
||||
)
|
||||
assert stats.compaction.fragments_removed == 0
|
||||
assert stats.compaction.fragments_added == 0
|
||||
|
||||
stats = await table.optimize(compaction_options={"target_rows_per_fragment": 3})
|
||||
assert stats.compaction.fragments_removed == 2
|
||||
assert stats.compaction.fragments_added == 1
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid compaction option: unknown"):
|
||||
await table.optimize(compaction_options={"unknown": 1})
|
||||
|
||||
|
||||
def test_optimize_compaction_options_sync(mem_db: DBConnection):
|
||||
table = mem_db.create_table("test", data=[{"x": 1}])
|
||||
with pytest.raises(ValueError, match="target_rows_per_fragment must be between"):
|
||||
table.optimize(compaction_options={"target_rows_per_fragment": 0})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("option", ["max_source_rows", "max_source_bytes"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_compaction_source_limits(
|
||||
mem_db_async: AsyncConnection, option: str
|
||||
):
|
||||
table = await mem_db_async.create_table("test", data=[{"x": 1}])
|
||||
await table.add([{"x": 2}])
|
||||
|
||||
stats = await table.optimize(
|
||||
compaction_options={
|
||||
"target_rows_per_fragment": 3,
|
||||
option: 1,
|
||||
}
|
||||
)
|
||||
assert stats.compaction.fragments_removed == 0
|
||||
assert stats.compaction.fragments_added == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_compaction_data_storage_version(
|
||||
mem_db_async: AsyncConnection,
|
||||
):
|
||||
table = await mem_db_async.create_table("test", data=[{"x": 1}])
|
||||
await table.add([{"x": 2}])
|
||||
|
||||
stats = await table.optimize(
|
||||
compaction_options={
|
||||
"target_rows_per_fragment": 3,
|
||||
"data_storage_version": "2.2",
|
||||
}
|
||||
)
|
||||
assert stats.compaction.fragments_removed == 2
|
||||
assert stats.compaction.fragments_added == 1
|
||||
|
||||
table = await mem_db_async.create_table("cross_family", data=[{"x": 1}])
|
||||
await table.add([{"x": 2}])
|
||||
with pytest.raises(ValueError, match="V1 and V2 storage versions cannot be mixed"):
|
||||
await table.optimize(
|
||||
compaction_options={
|
||||
"target_rows_per_fragment": 3,
|
||||
"data_storage_version": "legacy",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_compaction_excluded_fragments(mem_db_async: AsyncConnection):
|
||||
table = await mem_db_async.create_table("test", data=[{"x": 1}])
|
||||
await table.add([{"x": 2}])
|
||||
|
||||
stats = await table.optimize(
|
||||
compaction_options={
|
||||
"target_rows_per_fragment": 3,
|
||||
"excluded_fragment_ids": [0],
|
||||
}
|
||||
)
|
||||
assert stats.compaction.fragments_removed == 0
|
||||
assert stats.compaction.fragments_added == 0
|
||||
|
||||
stats = await table.optimize(compaction_options={"target_rows_per_fragment": 3})
|
||||
assert stats.compaction.fragments_removed == 2
|
||||
assert stats.compaction.fragments_added == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("option", "value", "message"),
|
||||
[
|
||||
("target_rows_per_fragment", 0, "must be between 1 and 4294967295"),
|
||||
("max_rows_per_group", 0, "must be between 1 and 4294967295"),
|
||||
("batch_size", 0, "must be between 1 and 4294967295"),
|
||||
("num_threads", 0, "must be greater than 0"),
|
||||
("target_rows_per_fragment", 2**32, "must be between 1 and 4294967295"),
|
||||
("max_rows_per_group", 2**32, "must be between 1 and 4294967295"),
|
||||
("batch_size", 2**32, "must be between 1 and 4294967295"),
|
||||
("io_buffer_size", 2**63, "must be at most 9223372036854775807"),
|
||||
("max_source_rows", 0, "must be greater than 0"),
|
||||
("max_source_bytes", 0, "must be greater than 0"),
|
||||
("data_storage_version", "invalid", "Unknown Lance storage version: invalid"),
|
||||
(
|
||||
"excluded_fragment_ids",
|
||||
[-1],
|
||||
"must contain values between 0 and 4294967295",
|
||||
),
|
||||
(
|
||||
"excluded_fragment_ids",
|
||||
[2**32],
|
||||
"must contain values between 0 and 4294967295",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_compaction_options_validation(
|
||||
mem_db_async: AsyncConnection, option: str, value: Any, message: str
|
||||
):
|
||||
table = await mem_db_async.create_table("test", data=[{"x": 1}])
|
||||
with pytest.raises(ValueError, match=message):
|
||||
await table.optimize(compaction_options={option: value})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_optimize_delete_unverified(tmp_db_async: AsyncConnection, tmp_path):
|
||||
table = await tmp_db_async.create_table(
|
||||
|
||||
Reference in New Issue
Block a user