mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
feat(functions): expose retry policies in Python and Rust (#4873)
Expose `FunctionRetryPolicy` on Function declarations and refresh overrides, together with `error_policy` and recorded attempt counts. Python uses the Rust validation rules before sending a request; local refreshes reject these Enterprise options. The policy bounds calls and backoff, supports class/message/provider-code matching, and can disable retries with `max_attempts=1`. Batch retry and skip operate on the whole failed call. Requires Enterprise server support for the new policy. Validation: rebuilt the Python extension; 47 targeted Python tests and 10 Rust tests passed. Python `make check`, Rust formatting, and `cargo clippy --features remote --tests --examples -- -D warnings` passed. The additional repository-root Ruff sweep reports existing errors under `docs/web-tests`; the Python workspace checks pass. Draft user guidance covers the supporting server requirement, safe repetition of partial work and whole-batch exhaustion. The guide sections stay hidden until release. Vale 3.24.0 passed for the added draft prose; no paired deployed SDK/server retry acceptance was run. Companion server: https://github.com/lancedb/sophon/pull/9812. BREAKING CHANGE: Rust callers constructing `FunctionRegistrationRequest` or `FunctionErrorRecord` with struct literals must supply the new `retry_policy` or `attempts` field, respectively.
This commit is contained in:
@@ -42,6 +42,7 @@ from .functions import (
|
||||
FunctionBinding as FunctionBinding,
|
||||
FunctionDefinition as FunctionDefinition,
|
||||
FunctionRegistrationRequest as FunctionRegistrationRequest,
|
||||
FunctionRetryPolicy as FunctionRetryPolicy,
|
||||
FunctionVersion as FunctionVersion,
|
||||
HuggingFaceModel as HuggingFaceModel,
|
||||
PythonRuntimeSpec as PythonRuntimeSpec,
|
||||
|
||||
@@ -617,6 +617,8 @@ class Table:
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[str] = None,
|
||||
error_policy: Optional[str] = None,
|
||||
) -> RefreshColumnResult: ...
|
||||
async def refresh_column_async(
|
||||
self,
|
||||
@@ -625,6 +627,8 @@ class Table:
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[str] = None,
|
||||
error_policy: Optional[str] = None,
|
||||
) -> Job: ...
|
||||
async def function_errors(
|
||||
self,
|
||||
@@ -1008,6 +1012,7 @@ class RefreshColumnResult:
|
||||
version: int
|
||||
|
||||
class FunctionErrorRecord:
|
||||
attempts: Optional[int]
|
||||
job_id: str
|
||||
fragment_id: int
|
||||
row_offset: Optional[int]
|
||||
@@ -1206,3 +1211,5 @@ class AuthzObject:
|
||||
) -> AuthzObject: ...
|
||||
def __str__(self) -> str: ...
|
||||
def __repr__(self) -> str: ...
|
||||
|
||||
def validate_function_retry_policy(json: str) -> None: ...
|
||||
|
||||
@@ -213,6 +213,45 @@ class _OpenRemoteValue(_RemoteValue):
|
||||
return set((self.__pydantic_extra__ or {}).keys())
|
||||
|
||||
|
||||
class FunctionRetryPolicy(_RemoteValue):
|
||||
"""Bound retries of selected errors inside one Enterprise Function call.
|
||||
|
||||
``max_attempts`` includes the first call. Matchers are alternatives;
|
||||
``transient=True`` selects connection/timeouts, HTTP 429/503 and the
|
||||
provider codes ``SlowDown`` and ``ServiceUnavailable``. Exception names
|
||||
may be qualified, and subclasses and wrapped causes also match.
|
||||
|
||||
Backoff doubles with full jitter, bounded by ``max_backoff_seconds``.
|
||||
Retry-After above that bound ends retries. A failed batch is retried as
|
||||
one call: under ``skip``, exhaustion nulls every row in that call.
|
||||
Completed calls are never repeated. Only opt in when repeating partial
|
||||
work is safe; this does not make external side effects exactly once.
|
||||
|
||||
>>> policy = FunctionRetryPolicy(transient=True, max_attempts=3)
|
||||
>>> policy.max_attempts
|
||||
3
|
||||
>>> disabled = FunctionRetryPolicy(max_attempts=1)
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", frozen=True)
|
||||
|
||||
max_attempts: Annotated[int, Field(strict=True)] = 3
|
||||
initial_backoff_seconds: Annotated[float, Field(strict=True)] = 0.2
|
||||
max_backoff_seconds: Annotated[float, Field(strict=True)] = 30.0
|
||||
exception_types: tuple[str, ...] = ()
|
||||
message_pattern: Optional[str] = None
|
||||
error_codes: tuple[str, ...] = ()
|
||||
transient: Annotated[bool, Field(strict=True)] = False
|
||||
honor_retry_after: Annotated[bool, Field(strict=True)] = True
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_policy(self):
|
||||
from ._lancedb import validate_function_retry_policy
|
||||
|
||||
validate_function_retry_policy(self.model_dump_json())
|
||||
return self
|
||||
|
||||
|
||||
class FunctionArtifact(_RemoteValue):
|
||||
"""Content-addressed Python artifact identity."""
|
||||
|
||||
@@ -653,6 +692,7 @@ class FunctionRegistrationRequest(_RemoteValue):
|
||||
runtime: PythonRuntimeSpec
|
||||
batch_size: Optional[int] = None
|
||||
row_timeout_seconds: Optional[float] = None
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None
|
||||
secret_bindings: tuple[SecretBinding, ...] = ()
|
||||
models: tuple[HuggingFaceModel, ...] = ()
|
||||
|
||||
@@ -2108,6 +2148,7 @@ class FunctionDefinition:
|
||||
python_version: Optional[str],
|
||||
batch_size: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
gpu: bool = False,
|
||||
gpu_class: Optional[str] = None,
|
||||
conda: tuple[str, ...] = (),
|
||||
@@ -2254,6 +2295,7 @@ class FunctionDefinition:
|
||||
runtime=runtime,
|
||||
batch_size=batch_size,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
retry_policy=retry_policy,
|
||||
models=models,
|
||||
)
|
||||
# A class's own __dict__ holds its methods; copying it would shadow
|
||||
@@ -2351,6 +2393,7 @@ def function(
|
||||
python_version: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
gpu: bool = False,
|
||||
gpu_class: Optional[str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
@@ -2373,6 +2416,7 @@ def function(
|
||||
python_version: Optional[str] = None,
|
||||
batch_size: Optional[int] = None,
|
||||
row_timeout_seconds: Optional[float] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
gpu: bool = False,
|
||||
gpu_class: Optional[str] = None,
|
||||
conda: tuple[str, ...] | list[str] = (),
|
||||
@@ -2504,6 +2548,11 @@ def function(
|
||||
scalar Functions and one batch for vectorized Functions. This does not
|
||||
govern class initialization. When omitted, the service applies its
|
||||
default per-call budget.
|
||||
retry_policy : FunctionRetryPolicy, optional
|
||||
Retry selected failed calls inside the Function, with bounded attempts
|
||||
and backoff. A refresh may override this default. Retrying a partial
|
||||
call must be safe; completed calls are not repeated. For batch Functions,
|
||||
skip after exhaustion applies to the whole failed call.
|
||||
gpu : bool, default False
|
||||
Whether every remote execution requires a GPU. The execution platform
|
||||
selects one compatible GPU for each worker. The requirement is part of
|
||||
@@ -2631,6 +2680,7 @@ def function(
|
||||
python_version=python_version,
|
||||
batch_size=batch_size,
|
||||
row_timeout_seconds=row_timeout_seconds,
|
||||
retry_policy=retry_policy,
|
||||
gpu=gpu,
|
||||
gpu_class=gpu_class,
|
||||
conda=tuple(conda),
|
||||
@@ -2657,6 +2707,7 @@ __all__ = [
|
||||
"FunctionOutput",
|
||||
"FunctionParameter",
|
||||
"FunctionRegistrationRequest",
|
||||
"FunctionRetryPolicy",
|
||||
"FunctionResultField",
|
||||
"FunctionSignature",
|
||||
"FunctionImage",
|
||||
|
||||
@@ -48,7 +48,11 @@ from lancedb.index import (
|
||||
LabelList,
|
||||
)
|
||||
from lancedb.job import Job
|
||||
from lancedb.functions import FunctionApplication, RefreshColumnResult
|
||||
from lancedb.functions import (
|
||||
FunctionApplication,
|
||||
FunctionRetryPolicy,
|
||||
RefreshColumnResult,
|
||||
)
|
||||
from lancedb.remote.db import LOOP
|
||||
from lancedb.table import IndexConfigType, _maybe_add_fts_error_note
|
||||
import pyarrow as pa
|
||||
@@ -1137,6 +1141,8 @@ class RemoteTable(Table):
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
):
|
||||
return LOOP.run(
|
||||
self._table.refresh_column(
|
||||
@@ -1144,6 +1150,8 @@ class RemoteTable(Table):
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1154,6 +1162,8 @@ class RemoteTable(Table):
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> Job[RefreshColumnResult]:
|
||||
return Job(
|
||||
LOOP.run(
|
||||
@@ -1162,6 +1172,8 @@ class RemoteTable(Table):
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@@ -82,6 +82,7 @@ from .index import (
|
||||
)
|
||||
from .expr import Expr
|
||||
from .functions import (
|
||||
FunctionRetryPolicy,
|
||||
FunctionApplication,
|
||||
RefreshColumnResult as RefreshColumnJobResult,
|
||||
)
|
||||
@@ -3166,6 +3167,8 @@ class Table(ABC):
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> "RefreshColumnResult":
|
||||
"""
|
||||
Fill the rows of a computed column that hold no value yet.
|
||||
@@ -3199,6 +3202,13 @@ class Table(ABC):
|
||||
Positive aggregate GPU budget for Function workers. None leaves
|
||||
the server's configured budget unchanged. LanceDB Enterprise
|
||||
only.
|
||||
retry_policy: FunctionRetryPolicy, optional
|
||||
Override retries of failed user-code calls. Only use when repeating
|
||||
partial work is safe. Set max_attempts=1 to disable retries.
|
||||
Enterprise Functions only; local refreshes reject this option.
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -3215,6 +3225,8 @@ class Table(ABC):
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> Job[RefreshColumnJobResult]:
|
||||
"""
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
@@ -3247,6 +3259,13 @@ class Table(ABC):
|
||||
Positive aggregate GPU budget for Function workers. None leaves
|
||||
the server's configured budget unchanged. LanceDB Enterprise
|
||||
only.
|
||||
retry_policy: FunctionRetryPolicy, optional
|
||||
Override retries of failed user-code calls. Only use when repeating
|
||||
partial work is safe. Set max_attempts=1 to disable retries.
|
||||
Enterprise Functions only; local refreshes reject this option.
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -5358,6 +5377,8 @@ class LanceTable(Table):
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> "RefreshColumnResult":
|
||||
"""Fill a computed column's unfilled rows and recompute those whose
|
||||
inputs changed. See
|
||||
@@ -5368,6 +5389,8 @@ class LanceTable(Table):
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -5378,6 +5401,8 @@ class LanceTable(Table):
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> Job[RefreshColumnJobResult]:
|
||||
"""Fill a computed column's unfilled rows and recompute those whose
|
||||
inputs changed, returning a handle to the refresh job. See
|
||||
@@ -5390,6 +5415,8 @@ class LanceTable(Table):
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=retry_policy,
|
||||
error_policy=error_policy,
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -7745,6 +7772,8 @@ class AsyncTable:
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> RefreshColumnResult:
|
||||
"""
|
||||
Fill the rows of a computed column that hold no value yet.
|
||||
@@ -7778,6 +7807,13 @@ class AsyncTable:
|
||||
Positive aggregate GPU budget for Function workers. None leaves
|
||||
the server's configured budget unchanged. LanceDB Enterprise
|
||||
only.
|
||||
retry_policy: FunctionRetryPolicy, optional
|
||||
Override retries of failed user-code calls. Only use when repeating
|
||||
partial work is safe. Set max_attempts=1 to disable retries.
|
||||
Enterprise Functions only; local refreshes reject this option.
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -7789,6 +7825,10 @@ class AsyncTable:
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=None
|
||||
if retry_policy is None
|
||||
else FunctionRetryPolicy.model_validate(retry_policy).model_dump_json(),
|
||||
error_policy=error_policy,
|
||||
)
|
||||
|
||||
async def function_errors(
|
||||
@@ -7887,6 +7927,8 @@ class AsyncTable:
|
||||
batch_size: Optional[int] = None,
|
||||
gpu_class: Optional[str] = None,
|
||||
gpu_count: Optional[int] = None,
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None,
|
||||
error_policy: Optional[Literal["fail", "skip"]] = None,
|
||||
) -> AsyncJob[RefreshColumnJobResult]:
|
||||
"""
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
@@ -7919,6 +7961,13 @@ class AsyncTable:
|
||||
Positive aggregate GPU budget for Function workers. None leaves
|
||||
the server's configured budget unchanged. LanceDB Enterprise
|
||||
only.
|
||||
retry_policy: FunctionRetryPolicy, optional
|
||||
Override retries of failed user-code calls. Only use when repeating
|
||||
partial work is safe. Set max_attempts=1 to disable retries.
|
||||
Enterprise Functions only; local refreshes reject this option.
|
||||
error_policy: {"fail", "skip"}, optional
|
||||
Action after failure or retry exhaustion. Skip records the error
|
||||
and attempt count, subject to the server's failure limits.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -7947,6 +7996,10 @@ class AsyncTable:
|
||||
batch_size=batch_size,
|
||||
gpu_class=gpu_class,
|
||||
gpu_count=gpu_count,
|
||||
retry_policy=None
|
||||
if retry_policy is None
|
||||
else FunctionRetryPolicy.model_validate(retry_policy).model_dump_json(),
|
||||
error_policy=error_policy,
|
||||
),
|
||||
RefreshColumnJobResult.from_json,
|
||||
)
|
||||
|
||||
@@ -159,6 +159,34 @@ def test_function_execution_controls_apply_to_vectorized_functions():
|
||||
assert request["row_timeout_seconds"] == 1.25
|
||||
|
||||
|
||||
def test_function_retry_policy_is_part_of_registration():
|
||||
policy = lancedb.FunctionRetryPolicy(transient=True, max_attempts=4)
|
||||
definition = function(retry_policy=policy)(normalize_score.__wrapped__)
|
||||
request = json.loads(definition.registration_request.to_canonical_json())
|
||||
assert request["retry_policy"] == {"max_attempts": 4, "transient": True}
|
||||
assert lancedb.FunctionRetryPolicy(max_attempts=1).max_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
{"max_attempts": 0, "transient": True},
|
||||
{"max_attempts": 101, "transient": True},
|
||||
{"max_attempts": True},
|
||||
{"max_attempts": 3},
|
||||
{"transient": True, "initial_backoff_seconds": -1},
|
||||
{"transient": True, "max_backoff_seconds": float("inf")},
|
||||
{"transient": True, "max_backoff_seconds": 0},
|
||||
{"message_pattern": "["},
|
||||
{"exception_types": [""]},
|
||||
{"transient": True, "typo": 1},
|
||||
],
|
||||
)
|
||||
def test_function_retry_policy_rejects_invalid_options(options):
|
||||
with pytest.raises(ValueError):
|
||||
lancedb.FunctionRetryPolicy(**options)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("options", "field"),
|
||||
[
|
||||
|
||||
@@ -1825,6 +1825,11 @@ def test_remote_create_index_async_returns_job():
|
||||
{"batch_size": 1},
|
||||
{"batch_size": 8},
|
||||
{"gpu_class": "a10g", "gpu_count": 2},
|
||||
{
|
||||
"retry_policy": {"transient": True, "max_attempts": 3},
|
||||
"error_policy": "skip",
|
||||
},
|
||||
{"retry_policy": {"max_attempts": 1}, "error_policy": "fail"},
|
||||
],
|
||||
)
|
||||
async def test_remote_refresh_async_returns_typed_terminal_result(
|
||||
@@ -1849,6 +1854,12 @@ async def test_remote_refresh_async_returns_typed_terminal_result(
|
||||
expected["gpu_class"] = refresh_kwargs["gpu_class"]
|
||||
if refresh_kwargs.get("gpu_count") is not None:
|
||||
expected["gpu_count"] = refresh_kwargs["gpu_count"]
|
||||
if refresh_kwargs.get("retry_policy") is not None:
|
||||
expected["retry_policy"] = lancedb.FunctionRetryPolicy.model_validate(
|
||||
refresh_kwargs["retry_policy"]
|
||||
).model_dump(mode="json")
|
||||
if refresh_kwargs.get("error_policy") is not None:
|
||||
expected["error_policy"] = refresh_kwargs["error_policy"]
|
||||
assert json.loads(body) == expected
|
||||
request.send_response(202)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
@@ -2010,10 +2021,12 @@ def test_remote_refresh_async_forwards_progress_from_submitted_handle():
|
||||
({"gpu_class": "a_b"}, "gpu_class"),
|
||||
({"gpu_count": 0}, "gpu_count"),
|
||||
({"gpu_count": -1}, "gpu_count"),
|
||||
({"retry_policy": {"transient": True, "max_attempts": 0}}, "retry_policy"),
|
||||
({"error_policy": "retry"}, "error_policy"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_remote_refresh_rejects_invalid_gpu_options_before_request(
|
||||
async def test_remote_refresh_rejects_invalid_options_before_request(
|
||||
refresh_kwargs, message
|
||||
):
|
||||
backfill_requests = []
|
||||
@@ -2062,7 +2075,8 @@ async def test_remote_refresh_rejects_invalid_gpu_options_before_request(
|
||||
assert backfill_requests == []
|
||||
|
||||
|
||||
def test_remote_function_errors_lists_the_rows_a_refresh_skipped():
|
||||
@pytest.mark.parametrize("attempts", [None, 3])
|
||||
def test_remote_function_errors_lists_the_rows_a_refresh_skipped(attempts):
|
||||
listing = {
|
||||
"records": [
|
||||
{
|
||||
@@ -2077,6 +2091,7 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped():
|
||||
"table_version": 11,
|
||||
"error_type": "ValueError",
|
||||
"error_message": "bad input 'x'",
|
||||
"attempts": attempts,
|
||||
"traceback": (
|
||||
"Traceback (most recent call last):\nValueError: bad input 'x'"
|
||||
),
|
||||
@@ -2127,6 +2142,7 @@ def test_remote_function_errors_lists_the_rows_a_refresh_skipped():
|
||||
assert errors.records[0].row_id == 42
|
||||
assert errors.records[0].row_address == 12884901897
|
||||
assert errors.records[0].function_version == "2"
|
||||
assert errors.records[0].attempts == attempts
|
||||
assert errors.records[0].traceback == (
|
||||
"Traceback (most recent call last):\nValueError: bad input 'x'"
|
||||
)
|
||||
|
||||
@@ -5984,6 +5984,14 @@ def test_refresh_column_rejects_invalid_batch_size(tmp_path, method, batch_size)
|
||||
getattr(table, method)("x", batch_size=batch_size)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method", ["refresh_column", "refresh_column_async"])
|
||||
def test_local_refresh_rejects_function_retry_policy(tmp_path, method):
|
||||
table = lancedb.connect(tmp_path).create_table("rows", [{"x": 1}])
|
||||
policy = lancedb.FunctionRetryPolicy(transient=True)
|
||||
with pytest.raises(NotImplementedError, match="Enterprise Function refresh"):
|
||||
getattr(table, method)("x", retry_policy=policy)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("method", ["refresh_column", "refresh_column_async"])
|
||||
@pytest.mark.parametrize("batch_size", [0, -1])
|
||||
|
||||
@@ -145,6 +145,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(table::tokenize, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(util::validate_function_retry_policy, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(util::parse_field_path, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(expr_col, m)?)?;
|
||||
|
||||
+26
-4
@@ -46,8 +46,22 @@ fn refresh_column_options(
|
||||
batch_size: Option<i64>,
|
||||
gpu_class: Option<String>,
|
||||
gpu_count: Option<i64>,
|
||||
retry_policy: Option<String>,
|
||||
error_policy: Option<String>,
|
||||
) -> PyResult<RefreshColumnOptions> {
|
||||
let mut options = RefreshColumnOptions::default();
|
||||
options.retry_policy = retry_policy
|
||||
.map(|json| {
|
||||
serde_json::from_str(&json)
|
||||
.map_err(|error| PyValueError::new_err(format!("invalid retry_policy: {error}")))
|
||||
})
|
||||
.transpose()?;
|
||||
options.error_policy = error_policy
|
||||
.map(|value| {
|
||||
serde_json::from_value(serde_json::Value::String(value))
|
||||
.map_err(|error| PyValueError::new_err(format!("invalid error_policy: {error}")))
|
||||
})
|
||||
.transpose()?;
|
||||
options.batch_size = batch_size
|
||||
.map(usize::try_from)
|
||||
.transpose()
|
||||
@@ -611,6 +625,7 @@ impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
|
||||
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FunctionErrorRecord {
|
||||
pub attempts: Option<u32>,
|
||||
pub job_id: String,
|
||||
pub fragment_id: u64,
|
||||
pub row_offset: Option<u32>,
|
||||
@@ -653,6 +668,7 @@ impl FunctionErrorRecord {
|
||||
impl From<lancedb::function::FunctionErrorRecord> for FunctionErrorRecord {
|
||||
fn from(record: lancedb::function::FunctionErrorRecord) -> Self {
|
||||
Self {
|
||||
attempts: record.attempts,
|
||||
job_id: record.job_id,
|
||||
fragment_id: record.fragment_id,
|
||||
row_offset: record.row_offset,
|
||||
@@ -2126,15 +2142,18 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (column, *, batch_size=None, gpu_class=None, gpu_count=None))]
|
||||
#[pyo3(signature = (column, *, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None))]
|
||||
pub fn refresh_column(
|
||||
self_: PyRef<'_, Self>,
|
||||
column: String,
|
||||
batch_size: Option<i64>,
|
||||
gpu_class: Option<String>,
|
||||
gpu_count: Option<i64>,
|
||||
retry_policy: Option<String>,
|
||||
error_policy: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let options = refresh_column_options(batch_size, gpu_class, gpu_count)?;
|
||||
let options =
|
||||
refresh_column_options(batch_size, gpu_class, gpu_count, retry_policy, error_policy)?;
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let result = inner
|
||||
@@ -2145,15 +2164,18 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (column, *, batch_size=None, gpu_class=None, gpu_count=None))]
|
||||
#[pyo3(signature = (column, *, batch_size=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None))]
|
||||
pub fn refresh_column_async(
|
||||
self_: PyRef<'_, Self>,
|
||||
column: String,
|
||||
batch_size: Option<i64>,
|
||||
gpu_class: Option<String>,
|
||||
gpu_count: Option<i64>,
|
||||
retry_policy: Option<String>,
|
||||
error_policy: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let options = refresh_column_options(batch_size, gpu_class, gpu_count)?;
|
||||
let options =
|
||||
refresh_column_options(batch_size, gpu_class, gpu_count, retry_policy, error_policy)?;
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let job = inner
|
||||
|
||||
@@ -70,3 +70,14 @@ pub fn parse_field_path(path: &str) -> PyResult<Vec<String>> {
|
||||
/// A wrapper around a LanceDB type to allow it to be used in Python
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PyLanceDB<T>(pub T);
|
||||
/// Validate the Function retry wire contract using the same rules as Rust APIs.
|
||||
#[pyo3::pyfunction]
|
||||
pub fn validate_function_retry_policy(json: &str) -> pyo3::PyResult<()> {
|
||||
let policy: lancedb::function::FunctionRetryPolicy =
|
||||
serde_json::from_str(json).map_err(|error| {
|
||||
pyo3::exceptions::PyValueError::new_err(format!("invalid retry_policy: {error}"))
|
||||
})?;
|
||||
policy
|
||||
.validate()
|
||||
.map_err(|error| pyo3::exceptions::PyValueError::new_err(error.to_string()))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user