mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
feat: expose the remaining refresh options on the SDKs (#4890)
The server's backfill request takes a row filter, failure limits, per-call timeouts and worker sizing, and SQL exposes all of them through `REFRESH COLUMN ... WHERE ... WITH (...)`, but the typed SDKs could send only batch_size, GPU placement and the retry and error policies. Anyone needing a filtered refresh or a bounded skip policy had to drop to SQL. This adds `where`, `max_failed_fraction`, `max_failed_rows`, `row_timeout_seconds`, `init_timeout_seconds`, `num_workers` and `max_executor_processes_per_worker` to Python, Rust and Node, passed through unchanged. `RefreshColumnOptions` now serializes straight into the request body, so the three SDKs share one producer. Node gets a `RefreshColumnOptions` object, which also carries the options it lacked before. A LanceDB Embedded refresh rejects every option other than batch_size and GPU placement, naming the option.
This commit is contained in:
Wyatt Alt
@@ -617,21 +617,35 @@ class Table:
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[str] = None,
|
||||
error_policy: Optional[str] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> RefreshColumnResult: ...
|
||||
async def refresh_column_async(
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[str] = None,
|
||||
error_policy: Optional[str] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> Job: ...
|
||||
async def function_errors(
|
||||
self,
|
||||
|
||||
@@ -1138,20 +1138,34 @@ class RemoteTable(Table):
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
):
|
||||
return LOOP.run(
|
||||
self._table.refresh_column(
|
||||
column,
|
||||
where=where,
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
max_failed_fraction=max_failed_fraction,
|
||||
max_failed_rows=max_failed_rows,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
init_timeout_seconds=init_timeout_seconds,
|
||||
num_workers=num_workers,
|
||||
max_executor_processes_per_worker=max_executor_processes_per_worker,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1159,21 +1173,35 @@ class RemoteTable(Table):
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> Job[RefreshColumnResult]:
|
||||
return Job(
|
||||
LOOP.run(
|
||||
self._table.refresh_column_async(
|
||||
column,
|
||||
where=where,
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
max_failed_fraction=max_failed_fraction,
|
||||
max_failed_rows=max_failed_rows,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
init_timeout_seconds=init_timeout_seconds,
|
||||
num_workers=num_workers,
|
||||
max_executor_processes_per_worker=max_executor_processes_per_worker,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -3164,11 +3164,18 @@ class Table(ABC):
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> "RefreshColumnResult":
|
||||
"""
|
||||
Fill the rows of a computed column that hold no value yet.
|
||||
@@ -3189,6 +3196,10 @@ class Table(ABC):
|
||||
----------
|
||||
column: str
|
||||
The name of the computed column to fill.
|
||||
where: str, optional
|
||||
SQL filter selecting the rows to refresh, as in ``REFRESH COLUMN
|
||||
... WHERE``. Unfilled rows outside it stay unfilled. LanceDB
|
||||
Enterprise only; local refreshes reject this option.
|
||||
batch_size: int, optional
|
||||
Maximum rows per Function call on the remote server. Must be
|
||||
greater than zero. None uses the server default. Has no effect on
|
||||
@@ -3209,6 +3220,22 @@ class Table(ABC):
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
max_failed_fraction: float, optional
|
||||
Under a skip policy, fail the job once more than this share of
|
||||
rows (0 to 1) has failed. LanceDB Enterprise only.
|
||||
max_failed_rows: int, optional
|
||||
Under a skip policy, fail the job once more than this many rows
|
||||
have failed. LanceDB Enterprise only.
|
||||
row_timeout_seconds: float, optional
|
||||
Deadline for one Function call. LanceDB Enterprise only.
|
||||
init_timeout_seconds: float, optional
|
||||
Deadline for Function initialization. LanceDB Enterprise only.
|
||||
num_workers: int, optional
|
||||
Worker count for an expression column; registered Function
|
||||
columns reject it. LanceDB Enterprise only.
|
||||
max_executor_processes_per_worker: int, optional
|
||||
Cap on executor processes per Function worker. LanceDB
|
||||
Enterprise only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -3222,11 +3249,18 @@ class Table(ABC):
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> Job[RefreshColumnJobResult]:
|
||||
"""
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
@@ -3245,6 +3279,10 @@ class Table(ABC):
|
||||
----------
|
||||
column: str
|
||||
The name of the computed column to fill.
|
||||
where: str, optional
|
||||
SQL filter selecting the rows to refresh, as in ``REFRESH COLUMN
|
||||
... WHERE``. Unfilled rows outside it stay unfilled. LanceDB
|
||||
Enterprise only; local refreshes reject this option.
|
||||
batch_size: int, optional
|
||||
Maximum rows per Function call on the remote server. Must be
|
||||
greater than zero. None uses the server default. Lower this for
|
||||
@@ -3266,6 +3304,22 @@ class Table(ABC):
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
max_failed_fraction: float, optional
|
||||
Under a skip policy, fail the job once more than this share of
|
||||
rows (0 to 1) has failed. LanceDB Enterprise only.
|
||||
max_failed_rows: int, optional
|
||||
Under a skip policy, fail the job once more than this many rows
|
||||
have failed. LanceDB Enterprise only.
|
||||
row_timeout_seconds: float, optional
|
||||
Deadline for one Function call. LanceDB Enterprise only.
|
||||
init_timeout_seconds: float, optional
|
||||
Deadline for Function initialization. LanceDB Enterprise only.
|
||||
num_workers: int, optional
|
||||
Worker count for an expression column; registered Function
|
||||
columns reject it. LanceDB Enterprise only.
|
||||
max_executor_processes_per_worker: int, optional
|
||||
Cap on executor processes per Function worker. LanceDB
|
||||
Enterprise only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -5374,11 +5428,18 @@ class LanceTable(Table):
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> "RefreshColumnResult":
|
||||
"""Fill a computed column's unfilled rows and recompute those whose
|
||||
inputs changed. See
|
||||
@@ -5386,11 +5447,18 @@ class LanceTable(Table):
|
||||
return LOOP.run(
|
||||
self._table.refresh_column(
|
||||
column,
|
||||
where=where,
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
max_failed_fraction=max_failed_fraction,
|
||||
max_failed_rows=max_failed_rows,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
init_timeout_seconds=init_timeout_seconds,
|
||||
num_workers=num_workers,
|
||||
max_executor_processes_per_worker=max_executor_processes_per_worker,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -5398,11 +5466,18 @@ class LanceTable(Table):
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> Job[RefreshColumnJobResult]:
|
||||
"""Fill a computed column's unfilled rows and recompute those whose
|
||||
inputs changed, returning a handle to the refresh job. See
|
||||
@@ -5412,11 +5487,18 @@ class LanceTable(Table):
|
||||
LOOP.run(
|
||||
self._table.refresh_column_async(
|
||||
column,
|
||||
where=where,
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
max_failed_fraction=max_failed_fraction,
|
||||
max_failed_rows=max_failed_rows,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
init_timeout_seconds=init_timeout_seconds,
|
||||
num_workers=num_workers,
|
||||
max_executor_processes_per_worker=max_executor_processes_per_worker,
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -7769,11 +7851,18 @@ class AsyncTable:
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> RefreshColumnResult:
|
||||
"""
|
||||
Fill the rows of a computed column that hold no value yet.
|
||||
@@ -7794,6 +7883,10 @@ class AsyncTable:
|
||||
----------
|
||||
column: str
|
||||
The name of the computed column to fill.
|
||||
where: str, optional
|
||||
SQL filter selecting the rows to refresh, as in ``REFRESH COLUMN
|
||||
... WHERE``. Unfilled rows outside it stay unfilled. LanceDB
|
||||
Enterprise only; local refreshes reject this option.
|
||||
batch_size: int, optional
|
||||
Maximum rows per Function call on the remote server. Must be
|
||||
greater than zero. None uses the server default. Has no effect on
|
||||
@@ -7814,6 +7907,22 @@ class AsyncTable:
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
max_failed_fraction: float, optional
|
||||
Under a skip policy, fail the job once more than this share of
|
||||
rows (0 to 1) has failed. LanceDB Enterprise only.
|
||||
max_failed_rows: int, optional
|
||||
Under a skip policy, fail the job once more than this many rows
|
||||
have failed. LanceDB Enterprise only.
|
||||
row_timeout_seconds: float, optional
|
||||
Deadline for one Function call. LanceDB Enterprise only.
|
||||
init_timeout_seconds: float, optional
|
||||
Deadline for Function initialization. LanceDB Enterprise only.
|
||||
num_workers: int, optional
|
||||
Worker count for an expression column; registered Function
|
||||
columns reject it. LanceDB Enterprise only.
|
||||
max_executor_processes_per_worker: int, optional
|
||||
Cap on executor processes per Function worker. LanceDB
|
||||
Enterprise only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -7822,6 +7931,7 @@ class AsyncTable:
|
||||
"""
|
||||
return await self._inner.refresh_column(
|
||||
column,
|
||||
where=where,
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
@@ -7829,6 +7939,12 @@ class AsyncTable:
|
||||
if retry_policy is None
|
||||
else FunctionRetryPolicy.model_validate(retry_policy).model_dump_json(),
|
||||
error_policy=error_policy,
|
||||
max_failed_fraction=max_failed_fraction,
|
||||
max_failed_rows=max_failed_rows,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
init_timeout_seconds=init_timeout_seconds,
|
||||
num_workers=num_workers,
|
||||
max_executor_processes_per_worker=max_executor_processes_per_worker,
|
||||
)
|
||||
|
||||
async def function_errors(
|
||||
@@ -7924,11 +8040,18 @@ class AsyncTable:
|
||||
self,
|
||||
column: str,
|
||||
*,
|
||||
where: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
max_failed_fraction: Optional[float] = None,
|
||||
max_failed_rows: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
init_timeout_seconds: Optional[float] = None,
|
||||
num_workers: Optional[int] = None,
|
||||
max_executor_processes_per_worker: Optional[int] = None,
|
||||
) -> AsyncJob[RefreshColumnJobResult]:
|
||||
"""
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
@@ -7947,6 +8070,10 @@ class AsyncTable:
|
||||
----------
|
||||
column: str
|
||||
The name of the computed column to fill.
|
||||
where: str, optional
|
||||
SQL filter selecting the rows to refresh, as in ``REFRESH COLUMN
|
||||
... WHERE``. Unfilled rows outside it stay unfilled. LanceDB
|
||||
Enterprise only; local refreshes reject this option.
|
||||
batch_size: int, optional
|
||||
Maximum rows per Function call on the remote server. Must be
|
||||
greater than zero. None uses the server default. Lower this for
|
||||
@@ -7968,6 +8095,22 @@ class AsyncTable:
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
max_failed_fraction: float, optional
|
||||
Under a skip policy, fail the job once more than this share of
|
||||
rows (0 to 1) has failed. LanceDB Enterprise only.
|
||||
max_failed_rows: int, optional
|
||||
Under a skip policy, fail the job once more than this many rows
|
||||
have failed. LanceDB Enterprise only.
|
||||
row_timeout_seconds: float, optional
|
||||
Deadline for one Function call. LanceDB Enterprise only.
|
||||
init_timeout_seconds: float, optional
|
||||
Deadline for Function initialization. LanceDB Enterprise only.
|
||||
num_workers: int, optional
|
||||
Worker count for an expression column; registered Function
|
||||
columns reject it. LanceDB Enterprise only.
|
||||
max_executor_processes_per_worker: int, optional
|
||||
Cap on executor processes per Function worker. LanceDB
|
||||
Enterprise only.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -7993,6 +8136,7 @@ class AsyncTable:
|
||||
return _typed_job(
|
||||
await self._inner.refresh_column_async(
|
||||
column,
|
||||
where=where,
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
@@ -8000,6 +8144,12 @@ class AsyncTable:
|
||||
if retry_policy is None
|
||||
else FunctionRetryPolicy.model_validate(retry_policy).model_dump_json(),
|
||||
error_policy=error_policy,
|
||||
max_failed_fraction=max_failed_fraction,
|
||||
max_failed_rows=max_failed_rows,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
init_timeout_seconds=init_timeout_seconds,
|
||||
num_workers=num_workers,
|
||||
max_executor_processes_per_worker=max_executor_processes_per_worker,
|
||||
),
|
||||
RefreshColumnJobResult.from_json,
|
||||
)
|
||||
|
||||
@@ -1868,6 +1868,16 @@ def test_remote_create_index_async_returns_job():
|
||||
"error_policy": "skip",
|
||||
},
|
||||
{"retry_policy": {"max_attempts": 1}, "error_policy": "fail"},
|
||||
{
|
||||
"where": "id % 2 = 0",
|
||||
"error_policy": "skip",
|
||||
"max_failed_fraction": 0.01,
|
||||
"max_failed_rows": 50,
|
||||
"row_timeout_seconds": 30,
|
||||
"init_timeout_seconds": 120,
|
||||
"num_workers": 4,
|
||||
"max_executor_processes_per_worker": 2,
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_remote_refresh_async_returns_typed_terminal_result(
|
||||
@@ -1898,6 +1908,17 @@ async def test_remote_refresh_async_returns_typed_terminal_result(
|
||||
).model_dump(mode="json")
|
||||
if refresh_kwargs.get("error_policy") is not None:
|
||||
expected["error_policy"] = refresh_kwargs["error_policy"]
|
||||
for key in (
|
||||
"where",
|
||||
"max_failed_fraction",
|
||||
"max_failed_rows",
|
||||
"row_timeout_seconds",
|
||||
"init_timeout_seconds",
|
||||
"num_workers",
|
||||
"max_executor_processes_per_worker",
|
||||
):
|
||||
if refresh_kwargs.get(key) is not None:
|
||||
expected[key] = refresh_kwargs[key]
|
||||
assert json.loads(body) == expected
|
||||
request.send_response(202)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
@@ -2061,6 +2082,16 @@ def test_remote_refresh_async_forwards_progress_from_submitted_handle():
|
||||
({"gpu_count": -1}, "gpu_count"),
|
||||
({"retry_policy": {"transient": True, "max_attempts": 0}}, "retry_policy"),
|
||||
({"error_policy": "retry"}, "error_policy"),
|
||||
({"where": " "}, "where"),
|
||||
({"max_failed_fraction": 1.5}, "max_failed_fraction"),
|
||||
({"max_failed_rows": -1}, "max_failed_rows"),
|
||||
({"row_timeout_seconds": 0}, "row_timeout_seconds"),
|
||||
({"init_timeout_seconds": -5}, "init_timeout_seconds"),
|
||||
({"num_workers": 0}, "num_workers"),
|
||||
(
|
||||
{"max_executor_processes_per_worker": -2},
|
||||
"max_executor_processes_per_worker",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -5985,11 +5985,28 @@ def test_refresh_column_rejects_invalid_batch_size(tmp_path, method, batch_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["refresh_column", "refresh_column_async"])
|
||||
def test_local_refresh_rejects_function_retry_policy(tmp_path, method):
|
||||
@pytest.mark.parametrize(
|
||||
"refresh_kwargs,option",
|
||||
[
|
||||
({"retry_policy": lancedb.FunctionRetryPolicy(transient=True)}, "retry_policy"),
|
||||
({"where": "x > 0"}, "where"),
|
||||
({"error_policy": "skip"}, "error_policy"),
|
||||
({"max_failed_fraction": 0.5}, "max_failed_fraction"),
|
||||
({"max_failed_rows": 10}, "max_failed_rows"),
|
||||
({"row_timeout_seconds": 30}, "row_timeout_seconds"),
|
||||
({"init_timeout_seconds": 60}, "init_timeout_seconds"),
|
||||
({"num_workers": 2}, "num_workers"),
|
||||
({"max_executor_processes_per_worker": 2}, "max_executor_processes_per_worker"),
|
||||
],
|
||||
)
|
||||
def test_local_refresh_rejects_enterprise_options(
|
||||
tmp_path, method, refresh_kwargs, option
|
||||
):
|
||||
table = lancedb.connect(tmp_path).create_table("rows", [{"x": 1}])
|
||||
policy = lancedb.FunctionRetryPolicy(transient=True)
|
||||
with pytest.raises(NotImplementedError, match="Enterprise Function refresh"):
|
||||
getattr(table, method)("x", retry_policy=policy)
|
||||
with pytest.raises(
|
||||
NotImplementedError, match=f"{option} requires a LanceDB Enterprise refresh"
|
||||
):
|
||||
getattr(table, method)("x", **refresh_kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
+96
-8
@@ -42,14 +42,62 @@ use pyo3::{
|
||||
|
||||
mod scannable;
|
||||
|
||||
fn refresh_column_options(
|
||||
/// Keyword arguments shared by `refresh_column` and `refresh_column_async`.
|
||||
struct RefreshColumnArgs {
|
||||
where_clause: Option<String>,
|
||||
batch_size: Option<i64>,
|
||||
gpu_class: Option<String>,
|
||||
gpu_count: Option<i64>,
|
||||
retry_policy: Option<String>,
|
||||
error_policy: Option<String>,
|
||||
) -> PyResult<RefreshColumnOptions> {
|
||||
max_failed_fraction: Option<f64>,
|
||||
max_failed_rows: Option<i64>,
|
||||
row_timeout_seconds: Option<f64>,
|
||||
init_timeout_seconds: Option<f64>,
|
||||
num_workers: Option<i64>,
|
||||
max_executor_processes_per_worker: Option<i64>,
|
||||
}
|
||||
|
||||
fn positive_count<T: TryFrom<i64>>(value: Option<i64>, name: &str) -> PyResult<Option<T>> {
|
||||
value
|
||||
.map(|v| {
|
||||
T::try_from(v)
|
||||
.ok()
|
||||
.filter(|_| v > 0)
|
||||
.ok_or_else(|| PyValueError::new_err(format!("{name} must be greater than zero")))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOptions> {
|
||||
let RefreshColumnArgs {
|
||||
where_clause,
|
||||
batch_size,
|
||||
gpu_class,
|
||||
gpu_count,
|
||||
retry_policy,
|
||||
error_policy,
|
||||
max_failed_fraction,
|
||||
max_failed_rows,
|
||||
row_timeout_seconds,
|
||||
init_timeout_seconds,
|
||||
num_workers,
|
||||
max_executor_processes_per_worker,
|
||||
} = args;
|
||||
let mut options = RefreshColumnOptions::default();
|
||||
options.where_clause = where_clause;
|
||||
options.max_failed_fraction = max_failed_fraction;
|
||||
options.max_failed_rows = max_failed_rows
|
||||
.map(u64::try_from)
|
||||
.transpose()
|
||||
.map_err(|_| PyValueError::new_err("max_failed_rows must be a non-negative integer"))?;
|
||||
options.row_timeout_seconds = row_timeout_seconds;
|
||||
options.init_timeout_seconds = init_timeout_seconds;
|
||||
options.num_workers = positive_count(num_workers, "num_workers")?;
|
||||
options.max_executor_processes_per_worker = positive_count(
|
||||
max_executor_processes_per_worker,
|
||||
"max_executor_processes_per_worker",
|
||||
)?;
|
||||
options.retry_policy = retry_policy
|
||||
.map(|json| {
|
||||
serde_json::from_str(&json)
|
||||
@@ -2142,18 +2190,38 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (column, *, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None))]
|
||||
#[pyo3(signature = (column, *, r#where=None, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn refresh_column(
|
||||
self_: PyRef<'_, Self>,
|
||||
column: String,
|
||||
r#where: Option<String>,
|
||||
batch_size: Option<i64>,
|
||||
gpu_class: Option<String>,
|
||||
gpu_count: Option<i64>,
|
||||
retry_policy: Option<String>,
|
||||
error_policy: Option<String>,
|
||||
max_failed_fraction: Option<f64>,
|
||||
max_failed_rows: Option<i64>,
|
||||
row_timeout_seconds: Option<f64>,
|
||||
init_timeout_seconds: Option<f64>,
|
||||
num_workers: Option<i64>,
|
||||
max_executor_processes_per_worker: Option<i64>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let options =
|
||||
refresh_column_options(batch_size, gpu_class, gpu_count, retry_policy, error_policy)?;
|
||||
let options = refresh_column_options(RefreshColumnArgs {
|
||||
where_clause: r#where,
|
||||
batch_size,
|
||||
gpu_class,
|
||||
gpu_count,
|
||||
retry_policy,
|
||||
error_policy,
|
||||
max_failed_fraction,
|
||||
max_failed_rows,
|
||||
row_timeout_seconds,
|
||||
init_timeout_seconds,
|
||||
num_workers,
|
||||
max_executor_processes_per_worker,
|
||||
})?;
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = inner
|
||||
@@ -2164,18 +2232,38 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (column, *, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None))]
|
||||
#[pyo3(signature = (column, *, r#where=None, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn refresh_column_async(
|
||||
self_: PyRef<'_, Self>,
|
||||
column: String,
|
||||
r#where: Option<String>,
|
||||
batch_size: Option<i64>,
|
||||
gpu_class: Option<String>,
|
||||
gpu_count: Option<i64>,
|
||||
retry_policy: Option<String>,
|
||||
error_policy: Option<String>,
|
||||
max_failed_fraction: Option<f64>,
|
||||
max_failed_rows: Option<i64>,
|
||||
row_timeout_seconds: Option<f64>,
|
||||
init_timeout_seconds: Option<f64>,
|
||||
num_workers: Option<i64>,
|
||||
max_executor_processes_per_worker: Option<i64>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let options =
|
||||
refresh_column_options(batch_size, gpu_class, gpu_count, retry_policy, error_policy)?;
|
||||
let options = refresh_column_options(RefreshColumnArgs {
|
||||
where_clause: r#where,
|
||||
batch_size,
|
||||
gpu_class,
|
||||
gpu_count,
|
||||
retry_policy,
|
||||
error_policy,
|
||||
max_failed_fraction,
|
||||
max_failed_rows,
|
||||
row_timeout_seconds,
|
||||
init_timeout_seconds,
|
||||
num_workers,
|
||||
max_executor_processes_per_worker,
|
||||
})?;
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let job = inner
|
||||
|
||||
Reference in New Issue
Block a user