feat(function): support literal inputs in column declarations (#5021)

Function-column declarations now preserve constants such as
`fn(video=col("mp4"), num_frames=lit(16))` without changing its Function
version. Constants travel in the application inputs; column bindings,
request input schemas, and dependency tracking contain only columns.
Changing a constant uses a new column binding rather than rebaking the
Function.

Persisted bindings with literals use metadata version 2, which older
readers reject. Column-only bindings retain version 1. Python preserves
scalar literal identity, including null and unsigned integers, and
rejects composed expressions or values outside the canonical literal
domain.

This SDK change requires matching server declaration and execution
support, after a dedicated SDK dependency update. Native validation
passed 48 Rust metadata tests, four request-transport cases and 57
rebuilt Python tests on `a42d680`; 32 pre-existing experimental tests
remained ignored. The test-assertion correction passed its exact
regression and strict Clippy on `53ff80a2`. On `993fae7d`, the three
remote metadata compatibility regressions, native Python public-export
test and strict Clippy passed. These review corrections use v3 for
unknown-version rejection, cover existing v2 literal bindings through
remote mutation guards, and export `LiteralInputBinding`. The unchanged
native implementation and verified extension are reused. Matching server
execution and original-workload acceptance are not established by this
PR.

Refs ENT-3667.
This commit is contained in:
Xuanwo
2026-11-14 20:59:52 +08:00
committed by GitHub
parent f6f8119ba4
commit 12b95b59ab
10 changed files with 576 additions and 58 deletions
+1 -1
View File
@@ -117,7 +117,7 @@ class PyExpr:
def expr_col(name: str) -> PyExpr: ...
def expr_lit(
value: Union[bool, int, float, str, bytes, date, datetime, Decimal],
value: Union[bool, int, float, str, bytes, date, datetime, Decimal, None],
) -> PyExpr: ...
def expr_func(name: str, args: List[PyExpr]) -> PyExpr: ...
+18 -5
View File
@@ -29,6 +29,8 @@ from lancedb._lancedb import PyExpr, expr_col, expr_lit, expr_func
__all__ = ["Expr", "col", "lit", "func"]
_NOT_LITERAL = object()
_STR_TO_PA_TYPE: dict = {
"bool": pa.bool_(),
"boolean": pa.bool_(),
@@ -92,9 +94,16 @@ class Expr:
# for dict keys / set membership.
__hash__ = None # type: ignore[assignment]
def __init__(self, inner: PyExpr, *, column_path: str | None = None) -> None:
def __init__(
self,
inner: PyExpr,
*,
column_path: str | None = None,
literal_value: object = _NOT_LITERAL,
) -> None:
self._inner = inner
self._column_path = column_path
self._literal_value = literal_value
def __bool__(self) -> bool:
# An Expr describes a predicate that the query engine evaluates per row,
@@ -301,14 +310,16 @@ def col(name: str) -> Expr:
return Expr(expr_col(name), column_path=name)
def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> Expr:
def lit(
value: Union[bool, int, float, str, bytes, date, datetime, Decimal, None],
) -> Expr:
"""Create a literal (constant) value expression.
Parameters
----------
value:
A Python ``bool``, ``int``, ``float``, ``str``, ``bytes``, ``date``,
``datetime``, or ``Decimal``.
``datetime``, ``Decimal``, or ``None`` for SQL NULL.
Examples
--------
@@ -316,10 +327,12 @@ def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) ->
>>> col("price") * lit(1.1)
Expr((price * 1.1))
"""
if not isinstance(value, (bool, int, float, str, bytes, date, datetime, Decimal)):
if value is not None and not isinstance(
value, (bool, int, float, str, bytes, date, datetime, Decimal)
):
raise TypeError(f"Unsupported literal type: {type(value).__name__}")
return Expr(expr_lit(value))
return Expr(expr_lit(value), literal_value=value)
def func(name: str, *args: ExprLike) -> Expr:
+55 -13
View File
@@ -158,11 +158,15 @@ def _validate_literal(value):
"Function integer literal is outside the canonical JSON range"
)
elif isinstance(value, Mapping):
if not all(isinstance(key, str) for key in value):
raise TypeError("Function literal object keys must be strings")
for child in value.values():
_validate_literal(child)
elif isinstance(value, (list, tuple)):
for child in value:
_validate_literal(child)
elif value is not None and not isinstance(value, (bool, str)):
raise TypeError(f"unsupported Function literal type: {type(value).__name__}")
return value
@@ -193,7 +197,7 @@ class _RemoteValue(BaseModel):
known = {}
for name, field in self.__class__.model_fields.items():
value = getattr(self, name)
if value is None:
if value is None and not field.is_required():
continue
if not field.is_required():
default_factory = field.default_factory
@@ -626,10 +630,17 @@ class FunctionVersion(_RemoteValue):
return _validate_row_timeout(value)
def __call__(self, **arguments: Any) -> FunctionApplication:
"""Bind this exact version to named table columns.
"""Bind this exact version to named table columns and constant inputs.
Every input must be a direct [lancedb.col][lancedb.expr.col]
reference. A Function that declares initialization fields takes their
reference or [lancedb.lit][lancedb.expr.lit] constant. Constants support
integers, strings, booleans, and null; floating-point, binary and
temporal literals are not in the canonical Function input contract.
The Enterprise service must support literal bindings and validates
constants against this exact version's input signature. To change a
constant, redeclare the column using the same Function version.
A Function that declares initialization fields takes their
values as further keyword arguments; they are constants of this
binding, fixed for every row, and every instance the binding runs is
created with them. A non-nullable field without a default must be
@@ -658,7 +669,7 @@ class FunctionVersion(_RemoteValue):
... "embedding": embed(text=col("body"), model="small", dimensions=512)
... })
"""
from lancedb.expr import Expr
from lancedb.expr import Expr, _NOT_LITERAL
parameters = tuple(parameter.name for parameter in self.signature.inputs)
fields = {field.name: field for field in self.signature.initialization}
@@ -682,17 +693,27 @@ class FunctionVersion(_RemoteValue):
bindings = []
for parameter in parameters:
value = arguments[parameter]
if not isinstance(value, Expr) or value._column_path is None:
if isinstance(value, Expr) and value._literal_value is not _NOT_LITERAL:
bindings.append(
ApplicationInput(
parameter=parameter,
kind="literal",
value=value._literal_value,
)
)
elif isinstance(value, Expr) and value._column_path is not None:
bindings.append(
ApplicationInput(
parameter=parameter,
kind="column",
value={"path": value._column_path},
)
)
else:
raise TypeError(
f"Function input {parameter!r} must be a direct col(...) reference"
f"Function input {parameter!r} must be a direct col(...) reference "
"or lit(...) constant"
)
bindings.append(
ApplicationInput(
parameter=parameter,
kind="column",
value={"path": value._column_path},
)
)
initialization = {}
for name in fields:
if name in arguments:
@@ -876,6 +897,18 @@ class InputBinding(_RemoteValue):
nullable: bool
class LiteralInputBinding(_RemoteValue):
"""Constant supplied to a Function parameter for every input row."""
parameter: str
value: Any
@field_validator("value")
@classmethod
def _validate_value(cls, value):
return _validate_literal(value)
class OutputMapping(_RemoteValue):
"""One stable result-field mapping."""
@@ -900,6 +933,7 @@ class FunctionBinding(_RemoteValue):
binding_id: str
function: FunctionVersionRef
inputs: tuple[InputBinding, ...]
literal_inputs: tuple[LiteralInputBinding, ...] = ()
outputs: tuple[OutputMapping, ...]
assignment: Optional[AssignmentMapping] = None
input_schema: Optional[Mapping[str, Any]] = None
@@ -907,6 +941,13 @@ class FunctionBinding(_RemoteValue):
initialization: Optional[str] = None
"""Standard base64 of the validated one-row Arrow IPC initialization stream."""
@model_validator(mode="after")
def _validate_input_parameters(self):
parameters = [input.parameter for input in (*self.inputs, *self.literal_inputs)]
if len(parameters) != len(set(parameters)):
raise ValueError("Function binding input parameters must be unique")
return self
def initialization_row(self) -> Optional[dict[str, Any]]:
"""The initialization values this binding creates instances with."""
if self.initialization is None:
@@ -2979,6 +3020,7 @@ __all__ = [
"FunctionVersionRef",
"HuggingFaceModel",
"InputBinding",
"LiteralInputBinding",
"OutputMapping",
"PythonEnvironmentSpec",
"PythonAdapterSpec",
@@ -8,7 +8,7 @@ from types import MappingProxyType
import pytest
from lancedb import col
from lancedb import col, lit
import lancedb.functions as functions
from lancedb.functions import (
FunctionApplication,
@@ -58,6 +58,7 @@ def assert_no_secret_values(value):
def test_public_function_values_are_in_api_reference():
assert "LiteralInputBinding" in functions.__all__
docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
rendered = docs.read_text()
for name in functions.__all__:
@@ -286,6 +287,67 @@ def test_function_version_binding_validates_names_and_direct_columns():
version(text=col("text").lower())
@pytest.mark.parametrize("constant", [16, 8, True, "prompt", None, 2**64 - 1])
def test_function_version_binds_direct_literals_without_changing_identity(constant):
value = job_result("remote_function_job.json")
value["signature"]["inputs"].append(
{
"name": "option",
"arrow_type": "bool"
if type(constant) is bool
else "utf8"
if isinstance(constant, str)
else "uint64"
if constant == 2**64 - 1
else "int64",
"nullable": True,
}
)
version = FunctionVersion.from_json(json.dumps(value))
expression = lit(constant)
application = version(text=col("body"), option=expression)
encoded = json.loads(application.to_canonical_json())
assert encoded["inputs"] == [
{"parameter": "text", "kind": "column", "value": {"path": "body"}},
{"parameter": "option", "kind": "literal", "value": constant},
]
assert application.function.version == version.version
assert application.function.object_id == version.object_id
assert application.function.manifest_digest == version.image.manifest_digest
assert FunctionApplication.from_json(application.to_canonical_json()) == application
if constant is None:
assert expression.to_sql() == "NULL"
if type(constant) is int:
assert expression.to_sql() == str(constant)
def test_function_literals_reject_expressions_and_unsupported_values():
version = FunctionVersion.from_json(
json.dumps(job_result("remote_function_job.json"))
)
with pytest.raises(TypeError, match="direct col"):
version(text=lit(16) + lit(8))
with pytest.raises(ValueError, match="floating-point Function literals"):
version(text=lit(1.5))
with pytest.raises(TypeError, match="unsupported Function literal type"):
version(text=lit(b"data"))
def test_function_literal_binding_preserves_null_and_rejects_duplicate_parameters():
value = json.loads((FIXTURES / "remote_function_binding.json").read_text())
value["inputs"] = value["inputs"][:1]
value["literal_inputs"] = [{"parameter": "body", "value": None}]
binding = FunctionBinding.from_json(json.dumps(value))
assert binding.literal_inputs[0].value is None
assert json.loads(binding.to_canonical_json())["literal_inputs"] == [
{"parameter": "body", "value": None}
]
assert FunctionBinding.from_json(binding.to_canonical_json()) == binding
value["literal_inputs"][0]["parameter"] = "title"
with pytest.raises(ValueError, match="input parameters must be unique"):
FunctionBinding.from_json(json.dumps(value))
def test_function_version_keeps_named_struct_outputs_in_one_application():
value = job_result("remote_function_job.json")
value["name"] = "text_features"
+7 -1
View File
@@ -164,9 +164,12 @@ pub fn expr_col(name: &str) -> PyExpr {
/// Create a literal value expression.
///
/// Supported Python types: `bool`, `int`, `float`, `str`, `bytes`, `date`,
/// `datetime`, `Decimal`.
/// `datetime`, `Decimal`, and `None` (SQL NULL).
#[pyfunction]
pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
if value.is_none() {
return Ok(PyExpr(df_lit(ScalarValue::Null)));
}
// bool must be checked before int because bool is a subclass of int in Python
if let Ok(b) = value.extract::<bool>() {
return Ok(PyExpr(df_lit(b)));
@@ -174,6 +177,9 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
if let Ok(i) = value.extract::<i64>() {
return Ok(PyExpr(df_lit(i)));
}
if let Ok(i) = value.extract::<u64>() {
return Ok(PyExpr(df_lit(i)));
}
// Decimal must be checked before f64: Python's Decimal implements __float__,
// so value.extract::<f64>() would succeed and silently truncate the value to
// f64, losing precision. Build a Decimal128 scalar to preserve it instead.