mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(function): support literal inputs in column declarations (#5021)
Function-column declarations now preserve constants such as
`fn(video=col("mp4"), num_frames=lit(16))` without changing its Function
version. Constants travel in the application inputs; column bindings,
request input schemas, and dependency tracking contain only columns.
Changing a constant uses a new column binding rather than rebaking the
Function.
Persisted bindings with literals use metadata version 2, which older
readers reject. Column-only bindings retain version 1. Python preserves
scalar literal identity, including null and unsigned integers, and
rejects composed expressions or values outside the canonical literal
domain.
This SDK change requires matching server declaration and execution
support, after a dedicated SDK dependency update. Native validation
passed 48 Rust metadata tests, four request-transport cases and 57
rebuilt Python tests on `a42d680`; 32 pre-existing experimental tests
remained ignored. The test-assertion correction passed its exact
regression and strict Clippy on `53ff80a2`. On `993fae7d`, the three
remote metadata compatibility regressions, native Python public-export
test and strict Clippy passed. These review corrections use v3 for
unknown-version rejection, cover existing v2 literal bindings through
remote mutation guards, and export `LiteralInputBinding`. The unchanged
native implementation and verified extension are reused. Matching server
execution and original-workload acceptance are not established by this
PR.
Refs ENT-3667.
This commit is contained in:
@@ -578,6 +578,8 @@ with BlobReader(read_range, len(payload)) as reader:
|
||||
|
||||
::: lancedb.functions.InputBinding
|
||||
|
||||
::: lancedb.functions.LiteralInputBinding
|
||||
|
||||
::: lancedb.functions.OutputMapping
|
||||
|
||||
::: lancedb.functions.AssignmentMapping
|
||||
|
||||
@@ -245,6 +245,52 @@ function application keeps its scalar or named-struct result in the named
|
||||
column. A bare named-struct application expands its result fields as one atomic
|
||||
binding, with aliases from `rename(columns=...)`.
|
||||
|
||||
{/* Draft for the Function literal binding release.
|
||||
|
||||
### Bind a constant input {#literal-inputs}
|
||||
|
||||
<Note>
|
||||
Literal inputs require an SDK that supports binding metadata version 2 and an
|
||||
Enterprise deployment with matching literal declaration and execution support.
|
||||
Upgrading the SDK alone does not enable this server capability. Older clients
|
||||
reject version 2 bindings; column-only bindings continue to use version 1.
|
||||
</Note>
|
||||
|
||||
Use `lit(...)` to pass the same value for every row while another parameter
|
||||
reads a column. For example, using the table above:
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
from lancedb import col, function, lit
|
||||
|
||||
|
||||
@function
|
||||
def multiply(value: float, factor: int) -> float:
|
||||
return value * factor
|
||||
|
||||
|
||||
multiplier = db.create_function_async(multiply).wait()
|
||||
table.add_columns(
|
||||
{"scaled": multiplier(value=col("value"), factor=lit(16))},
|
||||
auto_refresh_enabled=False,
|
||||
)
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
Supported scalar constants are booleans, strings, integers from `-(2**63)`
|
||||
through `2**64 - 1`, and `None` for a nullable parameter. Each value must match
|
||||
the parameter's declared type. Floating-point, binary, date, datetime and
|
||||
decimal constants are not supported in Function input bindings, even though
|
||||
`lit(...)` accepts them in other SQL expressions. Bind a direct literal;
|
||||
expressions such as `lit(16) + lit(8)` are not supported here.
|
||||
|
||||
The constant belongs to the column binding, so the Function version and its
|
||||
image stay the same. Only column inputs contribute table dependencies. Refresh
|
||||
`scaled` to fill it. To change the constant to `lit(8)`, drop and redeclare the
|
||||
column using the same `multiplier` version; the new binding needs a refresh.
|
||||
|
||||
*/}
|
||||
|
||||
### Bind more than one input, or fill more than one column {#multi}
|
||||
|
||||
Parameters bind by name, so a function that takes several arguments binds
|
||||
|
||||
@@ -117,7 +117,7 @@ class PyExpr:
|
||||
|
||||
def expr_col(name: str) -> PyExpr: ...
|
||||
def expr_lit(
|
||||
value: Union[bool, int, float, str, bytes, date, datetime, Decimal],
|
||||
value: Union[bool, int, float, str, bytes, date, datetime, Decimal, None],
|
||||
) -> PyExpr: ...
|
||||
def expr_func(name: str, args: List[PyExpr]) -> PyExpr: ...
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ from lancedb._lancedb import PyExpr, expr_col, expr_lit, expr_func
|
||||
|
||||
__all__ = ["Expr", "col", "lit", "func"]
|
||||
|
||||
_NOT_LITERAL = object()
|
||||
|
||||
_STR_TO_PA_TYPE: dict = {
|
||||
"bool": pa.bool_(),
|
||||
"boolean": pa.bool_(),
|
||||
@@ -92,9 +94,16 @@ class Expr:
|
||||
# for dict keys / set membership.
|
||||
__hash__ = None # type: ignore[assignment]
|
||||
|
||||
def __init__(self, inner: PyExpr, *, column_path: str | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
inner: PyExpr,
|
||||
*,
|
||||
column_path: str | None = None,
|
||||
literal_value: object = _NOT_LITERAL,
|
||||
) -> None:
|
||||
self._inner = inner
|
||||
self._column_path = column_path
|
||||
self._literal_value = literal_value
|
||||
|
||||
def __bool__(self) -> bool:
|
||||
# An Expr describes a predicate that the query engine evaluates per row,
|
||||
@@ -301,14 +310,16 @@ def col(name: str) -> Expr:
|
||||
return Expr(expr_col(name), column_path=name)
|
||||
|
||||
|
||||
def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> Expr:
|
||||
def lit(
|
||||
value: Union[bool, int, float, str, bytes, date, datetime, Decimal, None],
|
||||
) -> Expr:
|
||||
"""Create a literal (constant) value expression.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
value:
|
||||
A Python ``bool``, ``int``, ``float``, ``str``, ``bytes``, ``date``,
|
||||
``datetime``, or ``Decimal``.
|
||||
``datetime``, ``Decimal``, or ``None`` for SQL NULL.
|
||||
|
||||
Examples
|
||||
--------
|
||||
@@ -316,10 +327,12 @@ def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) ->
|
||||
>>> col("price") * lit(1.1)
|
||||
Expr((price * 1.1))
|
||||
"""
|
||||
if not isinstance(value, (bool, int, float, str, bytes, date, datetime, Decimal)):
|
||||
if value is not None and not isinstance(
|
||||
value, (bool, int, float, str, bytes, date, datetime, Decimal)
|
||||
):
|
||||
raise TypeError(f"Unsupported literal type: {type(value).__name__}")
|
||||
|
||||
return Expr(expr_lit(value))
|
||||
return Expr(expr_lit(value), literal_value=value)
|
||||
|
||||
|
||||
def func(name: str, *args: ExprLike) -> Expr:
|
||||
|
||||
@@ -158,11 +158,15 @@ def _validate_literal(value):
|
||||
"Function integer literal is outside the canonical JSON range"
|
||||
)
|
||||
elif isinstance(value, Mapping):
|
||||
if not all(isinstance(key, str) for key in value):
|
||||
raise TypeError("Function literal object keys must be strings")
|
||||
for child in value.values():
|
||||
_validate_literal(child)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for child in value:
|
||||
_validate_literal(child)
|
||||
elif value is not None and not isinstance(value, (bool, str)):
|
||||
raise TypeError(f"unsupported Function literal type: {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
@@ -193,7 +197,7 @@ class _RemoteValue(BaseModel):
|
||||
known = {}
|
||||
for name, field in self.__class__.model_fields.items():
|
||||
value = getattr(self, name)
|
||||
if value is None:
|
||||
if value is None and not field.is_required():
|
||||
continue
|
||||
if not field.is_required():
|
||||
default_factory = field.default_factory
|
||||
@@ -626,10 +630,17 @@ class FunctionVersion(_RemoteValue):
|
||||
return _validate_row_timeout(value)
|
||||
|
||||
def __call__(self, **arguments: Any) -> FunctionApplication:
|
||||
"""Bind this exact version to named table columns.
|
||||
"""Bind this exact version to named table columns and constant inputs.
|
||||
|
||||
Every input must be a direct [lancedb.col][lancedb.expr.col]
|
||||
reference. A Function that declares initialization fields takes their
|
||||
reference or [lancedb.lit][lancedb.expr.lit] constant. Constants support
|
||||
integers, strings, booleans, and null; floating-point, binary and
|
||||
temporal literals are not in the canonical Function input contract.
|
||||
The Enterprise service must support literal bindings and validates
|
||||
constants against this exact version's input signature. To change a
|
||||
constant, redeclare the column using the same Function version.
|
||||
|
||||
A Function that declares initialization fields takes their
|
||||
values as further keyword arguments; they are constants of this
|
||||
binding, fixed for every row, and every instance the binding runs is
|
||||
created with them. A non-nullable field without a default must be
|
||||
@@ -658,7 +669,7 @@ class FunctionVersion(_RemoteValue):
|
||||
... "embedding": embed(text=col("body"), model="small", dimensions=512)
|
||||
... })
|
||||
"""
|
||||
from lancedb.expr import Expr
|
||||
from lancedb.expr import Expr, _NOT_LITERAL
|
||||
|
||||
parameters = tuple(parameter.name for parameter in self.signature.inputs)
|
||||
fields = {field.name: field for field in self.signature.initialization}
|
||||
@@ -682,17 +693,27 @@ class FunctionVersion(_RemoteValue):
|
||||
bindings = []
|
||||
for parameter in parameters:
|
||||
value = arguments[parameter]
|
||||
if not isinstance(value, Expr) or value._column_path is None:
|
||||
if isinstance(value, Expr) and value._literal_value is not _NOT_LITERAL:
|
||||
bindings.append(
|
||||
ApplicationInput(
|
||||
parameter=parameter,
|
||||
kind="literal",
|
||||
value=value._literal_value,
|
||||
)
|
||||
)
|
||||
elif isinstance(value, Expr) and value._column_path is not None:
|
||||
bindings.append(
|
||||
ApplicationInput(
|
||||
parameter=parameter,
|
||||
kind="column",
|
||||
value={"path": value._column_path},
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Function input {parameter!r} must be a direct col(...) reference"
|
||||
f"Function input {parameter!r} must be a direct col(...) reference "
|
||||
"or lit(...) constant"
|
||||
)
|
||||
bindings.append(
|
||||
ApplicationInput(
|
||||
parameter=parameter,
|
||||
kind="column",
|
||||
value={"path": value._column_path},
|
||||
)
|
||||
)
|
||||
initialization = {}
|
||||
for name in fields:
|
||||
if name in arguments:
|
||||
@@ -876,6 +897,18 @@ class InputBinding(_RemoteValue):
|
||||
nullable: bool
|
||||
|
||||
|
||||
class LiteralInputBinding(_RemoteValue):
|
||||
"""Constant supplied to a Function parameter for every input row."""
|
||||
|
||||
parameter: str
|
||||
value: Any
|
||||
|
||||
@field_validator("value")
|
||||
@classmethod
|
||||
def _validate_value(cls, value):
|
||||
return _validate_literal(value)
|
||||
|
||||
|
||||
class OutputMapping(_RemoteValue):
|
||||
"""One stable result-field mapping."""
|
||||
|
||||
@@ -900,6 +933,7 @@ class FunctionBinding(_RemoteValue):
|
||||
binding_id: str
|
||||
function: FunctionVersionRef
|
||||
inputs: tuple[InputBinding, ...]
|
||||
literal_inputs: tuple[LiteralInputBinding, ...] = ()
|
||||
outputs: tuple[OutputMapping, ...]
|
||||
assignment: Optional[AssignmentMapping] = None
|
||||
input_schema: Optional[Mapping[str, Any]] = None
|
||||
@@ -907,6 +941,13 @@ class FunctionBinding(_RemoteValue):
|
||||
initialization: Optional[str] = None
|
||||
"""Standard base64 of the validated one-row Arrow IPC initialization stream."""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_input_parameters(self):
|
||||
parameters = [input.parameter for input in (*self.inputs, *self.literal_inputs)]
|
||||
if len(parameters) != len(set(parameters)):
|
||||
raise ValueError("Function binding input parameters must be unique")
|
||||
return self
|
||||
|
||||
def initialization_row(self) -> Optional[dict[str, Any]]:
|
||||
"""The initialization values this binding creates instances with."""
|
||||
if self.initialization is None:
|
||||
@@ -2979,6 +3020,7 @@ __all__ = [
|
||||
"FunctionVersionRef",
|
||||
"HuggingFaceModel",
|
||||
"InputBinding",
|
||||
"LiteralInputBinding",
|
||||
"OutputMapping",
|
||||
"PythonEnvironmentSpec",
|
||||
"PythonAdapterSpec",
|
||||
|
||||
@@ -8,7 +8,7 @@ from types import MappingProxyType
|
||||
|
||||
import pytest
|
||||
|
||||
from lancedb import col
|
||||
from lancedb import col, lit
|
||||
import lancedb.functions as functions
|
||||
from lancedb.functions import (
|
||||
FunctionApplication,
|
||||
@@ -58,6 +58,7 @@ def assert_no_secret_values(value):
|
||||
|
||||
|
||||
def test_public_function_values_are_in_api_reference():
|
||||
assert "LiteralInputBinding" in functions.__all__
|
||||
docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
|
||||
rendered = docs.read_text()
|
||||
for name in functions.__all__:
|
||||
@@ -286,6 +287,67 @@ def test_function_version_binding_validates_names_and_direct_columns():
|
||||
version(text=col("text").lower())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("constant", [16, 8, True, "prompt", None, 2**64 - 1])
|
||||
def test_function_version_binds_direct_literals_without_changing_identity(constant):
|
||||
value = job_result("remote_function_job.json")
|
||||
value["signature"]["inputs"].append(
|
||||
{
|
||||
"name": "option",
|
||||
"arrow_type": "bool"
|
||||
if type(constant) is bool
|
||||
else "utf8"
|
||||
if isinstance(constant, str)
|
||||
else "uint64"
|
||||
if constant == 2**64 - 1
|
||||
else "int64",
|
||||
"nullable": True,
|
||||
}
|
||||
)
|
||||
version = FunctionVersion.from_json(json.dumps(value))
|
||||
expression = lit(constant)
|
||||
application = version(text=col("body"), option=expression)
|
||||
encoded = json.loads(application.to_canonical_json())
|
||||
assert encoded["inputs"] == [
|
||||
{"parameter": "text", "kind": "column", "value": {"path": "body"}},
|
||||
{"parameter": "option", "kind": "literal", "value": constant},
|
||||
]
|
||||
assert application.function.version == version.version
|
||||
assert application.function.object_id == version.object_id
|
||||
assert application.function.manifest_digest == version.image.manifest_digest
|
||||
assert FunctionApplication.from_json(application.to_canonical_json()) == application
|
||||
if constant is None:
|
||||
assert expression.to_sql() == "NULL"
|
||||
if type(constant) is int:
|
||||
assert expression.to_sql() == str(constant)
|
||||
|
||||
|
||||
def test_function_literals_reject_expressions_and_unsupported_values():
|
||||
version = FunctionVersion.from_json(
|
||||
json.dumps(job_result("remote_function_job.json"))
|
||||
)
|
||||
with pytest.raises(TypeError, match="direct col"):
|
||||
version(text=lit(16) + lit(8))
|
||||
with pytest.raises(ValueError, match="floating-point Function literals"):
|
||||
version(text=lit(1.5))
|
||||
with pytest.raises(TypeError, match="unsupported Function literal type"):
|
||||
version(text=lit(b"data"))
|
||||
|
||||
|
||||
def test_function_literal_binding_preserves_null_and_rejects_duplicate_parameters():
|
||||
value = json.loads((FIXTURES / "remote_function_binding.json").read_text())
|
||||
value["inputs"] = value["inputs"][:1]
|
||||
value["literal_inputs"] = [{"parameter": "body", "value": None}]
|
||||
binding = FunctionBinding.from_json(json.dumps(value))
|
||||
assert binding.literal_inputs[0].value is None
|
||||
assert json.loads(binding.to_canonical_json())["literal_inputs"] == [
|
||||
{"parameter": "body", "value": None}
|
||||
]
|
||||
assert FunctionBinding.from_json(binding.to_canonical_json()) == binding
|
||||
value["literal_inputs"][0]["parameter"] = "title"
|
||||
with pytest.raises(ValueError, match="input parameters must be unique"):
|
||||
FunctionBinding.from_json(json.dumps(value))
|
||||
|
||||
|
||||
def test_function_version_keeps_named_struct_outputs_in_one_application():
|
||||
value = job_result("remote_function_job.json")
|
||||
value["name"] = "text_features"
|
||||
|
||||
+7
-1
@@ -164,9 +164,12 @@ pub fn expr_col(name: &str) -> PyExpr {
|
||||
/// Create a literal value expression.
|
||||
///
|
||||
/// Supported Python types: `bool`, `int`, `float`, `str`, `bytes`, `date`,
|
||||
/// `datetime`, `Decimal`.
|
||||
/// `datetime`, `Decimal`, and `None` (SQL NULL).
|
||||
#[pyfunction]
|
||||
pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
|
||||
if value.is_none() {
|
||||
return Ok(PyExpr(df_lit(ScalarValue::Null)));
|
||||
}
|
||||
// bool must be checked before int because bool is a subclass of int in Python
|
||||
if let Ok(b) = value.extract::<bool>() {
|
||||
return Ok(PyExpr(df_lit(b)));
|
||||
@@ -174,6 +177,9 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
|
||||
if let Ok(i) = value.extract::<i64>() {
|
||||
return Ok(PyExpr(df_lit(i)));
|
||||
}
|
||||
if let Ok(i) = value.extract::<u64>() {
|
||||
return Ok(PyExpr(df_lit(i)));
|
||||
}
|
||||
// Decimal must be checked before f64: Python's Decimal implements __float__,
|
||||
// so value.extract::<f64>() would succeed and silently truncate the value to
|
||||
// f64, losing precision. Build a Decimal128 scalar to preserve it instead.
|
||||
|
||||
@@ -111,7 +111,7 @@ fn from_json<T: DeserializeOwned>(json: &str) -> Result<T> {
|
||||
serde_json::from_str(json).map_err(invalid_json)
|
||||
}
|
||||
|
||||
fn validate_literal(value: &Value) -> Result<()> {
|
||||
pub(crate) fn validate_literal(value: &Value) -> Result<()> {
|
||||
match value {
|
||||
Value::Number(number) if number.is_f64() => Err(Error::InvalidInput {
|
||||
message: "floating-point Function literals are not part of the Slice 1 canonical wire contract"
|
||||
@@ -1018,6 +1018,18 @@ pub struct InputBinding {
|
||||
pub nullable: bool,
|
||||
}
|
||||
|
||||
/// Constant value bound to a Function parameter for every input row.
|
||||
///
|
||||
/// Values use the same canonical JSON domain as [`ApplicationInput`]. The
|
||||
/// Enterprise service validates their types against the exact Function version.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct LiteralInputBinding {
|
||||
/// Function parameter name, distinct from all column-bound parameters.
|
||||
pub parameter: String,
|
||||
/// Immutable canonical JSON value.
|
||||
pub value: Value,
|
||||
}
|
||||
|
||||
/// Ordered result-field to table-field mapping for a Function binding.
|
||||
///
|
||||
/// `nullable` describes the logical Function result. Physical computed-column
|
||||
@@ -1046,6 +1058,8 @@ pub struct FunctionBinding {
|
||||
binding_id: String,
|
||||
function: FunctionVersionRef,
|
||||
inputs: Vec<InputBinding>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
literal_inputs: Vec<LiteralInputBinding>,
|
||||
outputs: Vec<OutputMapping>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
assignment: Option<AssignmentMapping>,
|
||||
@@ -1076,6 +1090,30 @@ impl FunctionBinding {
|
||||
&self.inputs
|
||||
}
|
||||
|
||||
/// Constants supplied for every row; these are not table dependencies.
|
||||
pub fn literal_inputs(&self) -> &[LiteralInputBinding] {
|
||||
&self.literal_inputs
|
||||
}
|
||||
|
||||
pub(crate) fn validate_inputs(&self) -> Result<()> {
|
||||
let mut parameters = std::collections::HashSet::new();
|
||||
for parameter in self
|
||||
.inputs
|
||||
.iter()
|
||||
.map(|input| &input.parameter)
|
||||
.chain(self.literal_inputs.iter().map(|input| &input.parameter))
|
||||
{
|
||||
if !parameters.insert(parameter) {
|
||||
return Err(invalid_json(format!(
|
||||
"duplicate Function parameter '{parameter}'"
|
||||
)));
|
||||
}
|
||||
}
|
||||
self.literal_inputs
|
||||
.iter()
|
||||
.try_for_each(|input| validate_literal(&input.value))
|
||||
}
|
||||
|
||||
pub fn outputs(&self) -> &[OutputMapping] {
|
||||
&self.outputs
|
||||
}
|
||||
@@ -1098,7 +1136,20 @@ impl FunctionBinding {
|
||||
}
|
||||
}
|
||||
|
||||
impl_json!(FunctionBinding);
|
||||
impl FunctionBinding {
|
||||
/// Decode an immutable binding, validating its canonical input values.
|
||||
pub fn from_json(json: &str) -> Result<Self> {
|
||||
let binding: Self = from_json(json)?;
|
||||
binding.validate_inputs()?;
|
||||
Ok(binding)
|
||||
}
|
||||
|
||||
/// Encode this binding without dropping its constant inputs.
|
||||
pub fn to_canonical_json(&self) -> Result<String> {
|
||||
self.validate_inputs()?;
|
||||
canonical_json(self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable terminal result of an expression-backed or Function-backed column
|
||||
/// refresh [`crate::Job`].
|
||||
|
||||
@@ -10769,9 +10769,17 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::column_only(None)]
|
||||
#[case::sixteen(Some(serde_json::json!(16)))]
|
||||
#[case::eight(Some(serde_json::json!(8)))]
|
||||
#[case::null(Some(serde_json::Value::Null))]
|
||||
#[tokio::test]
|
||||
async fn test_add_scalar_function_column_sends_atomic_null_declaration() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
async fn test_add_scalar_function_column_sends_atomic_null_declaration(
|
||||
#[case] literal: Option<serde_json::Value>,
|
||||
) {
|
||||
let expected_literal = literal.clone();
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
@@ -10789,6 +10797,13 @@ mod tests {
|
||||
))
|
||||
.unwrap();
|
||||
expected["auto_refresh_enabled"] = serde_json::json!(true);
|
||||
if let Some(literal) = &expected_literal {
|
||||
expected["function"]["binding_metadata_version"] = serde_json::json!(2);
|
||||
expected["function"]["application"]["inputs"]
|
||||
.as_array_mut().unwrap().push(serde_json::json!({
|
||||
"parameter": "num_frames", "kind": "literal", "value": literal,
|
||||
}));
|
||||
}
|
||||
assert_eq!(actual, expected);
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
@@ -10798,7 +10813,7 @@ mod tests {
|
||||
path => panic!("Unexpected path: {path}"),
|
||||
}
|
||||
});
|
||||
let application = crate::function::FunctionApplication::from_json(
|
||||
let mut application: serde_json::Value = serde_json::from_str(
|
||||
r#"{
|
||||
"function":{"name":"embed","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
|
||||
"inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
|
||||
@@ -10806,6 +10821,18 @@ mod tests {
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
if let Some(literal) = literal {
|
||||
application["inputs"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.push(serde_json::json!({
|
||||
"parameter": "num_frames", "kind": "literal", "value": literal,
|
||||
}));
|
||||
}
|
||||
let application = crate::function::FunctionApplication::from_json(
|
||||
&serde_json::to_string(&application).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = table
|
||||
.add_columns()
|
||||
@@ -10934,19 +10961,64 @@ mod tests {
|
||||
|
||||
/// Only a column the binding uses is refused, and it is refused before
|
||||
/// any request goes out; the rest reach the server as usual.
|
||||
#[rstest]
|
||||
#[case::column_only(false)]
|
||||
#[case::literal(true)]
|
||||
#[tokio::test]
|
||||
async fn test_add_columns_scopes_to_the_columns_a_binding_uses() {
|
||||
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(describe_response(&fixture_bound_schema()))
|
||||
.unwrap(),
|
||||
"/v1/table/my_table/add_columns/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version":10}"#.to_string())
|
||||
.unwrap(),
|
||||
path => panic!("Unexpected path: {path}"),
|
||||
});
|
||||
async fn test_add_columns_scopes_to_the_columns_a_binding_uses(#[case] literal: bool) {
|
||||
let mut schema = fixture_bound_schema();
|
||||
if literal {
|
||||
let key = crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY;
|
||||
let mut metadata = schema.metadata().clone();
|
||||
let mut envelope: serde_json::Value =
|
||||
serde_json::from_str(metadata.get(key).unwrap()).unwrap();
|
||||
envelope["version"] = serde_json::json!(2);
|
||||
envelope["bindings"][0]["inputs"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.pop();
|
||||
envelope["bindings"][0]["literal_inputs"] =
|
||||
serde_json::json!([{"parameter": "body", "value": null}]);
|
||||
let binding =
|
||||
crate::function::FunctionBinding::from_json(&envelope["bindings"][0].to_string())
|
||||
.unwrap();
|
||||
let fields = schema
|
||||
.fields()
|
||||
.iter()
|
||||
.map(|field| {
|
||||
let mut field = field.as_ref().clone();
|
||||
if let Some(output) = binding
|
||||
.outputs()
|
||||
.iter()
|
||||
.find(|output| output.output_name == *field.name())
|
||||
{
|
||||
field = field.with_metadata(
|
||||
crate::table::computed_columns::function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
output.output_ordinal,
|
||||
&["title".into()],
|
||||
),
|
||||
);
|
||||
}
|
||||
field
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
metadata.insert(key.to_string(), envelope.to_string());
|
||||
schema = Schema::new_with_metadata(fields, metadata);
|
||||
}
|
||||
let allowed_schema = schema.clone();
|
||||
let table =
|
||||
Table::new_with_handler("my_table", move |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(describe_response(&allowed_schema))
|
||||
.unwrap(),
|
||||
"/v1/table/my_table/add_columns/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version":10}"#.to_string())
|
||||
.unwrap(),
|
||||
path => panic!("Unexpected path: {path}"),
|
||||
});
|
||||
table
|
||||
.add_columns()
|
||||
.computed("doubled", "spare * 2")
|
||||
@@ -10963,13 +11035,14 @@ mod tests {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(describe_response(&fixture_bound_schema()))
|
||||
.unwrap(),
|
||||
path => panic!("mutation request must not be sent: {path}"),
|
||||
});
|
||||
let table =
|
||||
Table::new_with_handler("my_table", move |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(describe_response(&schema))
|
||||
.unwrap(),
|
||||
path => panic!("mutation request must not be sent: {path}"),
|
||||
});
|
||||
for name in ["title", "search_text"] {
|
||||
let err = table
|
||||
.add_columns()
|
||||
@@ -11327,7 +11400,7 @@ mod tests {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":2,\"bindings\":[]}"}}}"#,
|
||||
r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":3,\"bindings\":[]}"}}}"#,
|
||||
)
|
||||
.unwrap(),
|
||||
path => panic!("mutation request must not be sent: {path}"),
|
||||
|
||||
@@ -203,6 +203,10 @@ pub const FUNCTION_BINDINGS_META_KEY: &str = "lancedb::function_bindings";
|
||||
/// Version of the schema-level Function binding envelope.
|
||||
pub const FUNCTION_BINDINGS_VERSION: u32 = 1;
|
||||
|
||||
/// Envelope version required when a Function binding contains constant inputs.
|
||||
/// Older readers reject this version instead of dropping those inputs.
|
||||
pub const FUNCTION_LITERAL_BINDINGS_VERSION: u32 = 2;
|
||||
|
||||
/// Field metadata key holding `{fragment id -> input signature}` as JSON,
|
||||
/// recorded by the refresh that last computed each fragment. Outside the
|
||||
/// declaration namespace on purpose: a declaration is immutable through
|
||||
@@ -329,6 +333,17 @@ struct FunctionBindingEnvelope {
|
||||
|
||||
/// Encode immutable Function bindings for schema-level persistence.
|
||||
pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result<String> {
|
||||
bindings
|
||||
.iter()
|
||||
.try_for_each(FunctionBinding::validate_inputs)?;
|
||||
let version = if bindings
|
||||
.iter()
|
||||
.any(|binding| !binding.literal_inputs().is_empty())
|
||||
{
|
||||
FUNCTION_LITERAL_BINDINGS_VERSION
|
||||
} else {
|
||||
FUNCTION_BINDINGS_VERSION
|
||||
};
|
||||
let bindings = bindings
|
||||
.iter()
|
||||
.map(serde_json::to_value)
|
||||
@@ -336,12 +351,10 @@ pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result<String
|
||||
.map_err(|e| Error::InvalidInput {
|
||||
message: format!("invalid Function binding metadata: {e}"),
|
||||
})?;
|
||||
serde_json::to_string(&FunctionBindingEnvelope {
|
||||
version: FUNCTION_BINDINGS_VERSION,
|
||||
bindings,
|
||||
})
|
||||
.map_err(|e| Error::InvalidInput {
|
||||
message: format!("invalid Function binding metadata: {e}"),
|
||||
serde_json::to_string(&FunctionBindingEnvelope { version, bindings }).map_err(|e| {
|
||||
Error::InvalidInput {
|
||||
message: format!("invalid Function binding metadata: {e}"),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -355,9 +368,12 @@ pub fn function_bindings(schema: &ArrowSchema) -> Result<Vec<FunctionBinding>> {
|
||||
.bindings
|
||||
.into_iter()
|
||||
.map(|binding| {
|
||||
serde_json::from_value(binding).map_err(|e| Error::InvalidInput {
|
||||
message: format!("invalid Function binding metadata: {e}"),
|
||||
})
|
||||
let binding: FunctionBinding =
|
||||
serde_json::from_value(binding).map_err(|e| Error::InvalidInput {
|
||||
message: format!("invalid Function binding metadata: {e}"),
|
||||
})?;
|
||||
binding.validate_inputs()?;
|
||||
Ok(binding)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -370,7 +386,10 @@ fn function_binding_envelope(schema: &ArrowSchema) -> Result<Option<FunctionBind
|
||||
serde_json::from_str(raw).map_err(|e| Error::InvalidInput {
|
||||
message: format!("invalid Function binding metadata: {e}"),
|
||||
})?;
|
||||
if envelope.version != FUNCTION_BINDINGS_VERSION {
|
||||
if !matches!(
|
||||
envelope.version,
|
||||
FUNCTION_BINDINGS_VERSION | FUNCTION_LITERAL_BINDINGS_VERSION
|
||||
) {
|
||||
return Err(Error::NotSupported {
|
||||
message: format!(
|
||||
"Function binding metadata version {} is not supported by this client",
|
||||
@@ -378,6 +397,18 @@ fn function_binding_envelope(schema: &ArrowSchema) -> Result<Option<FunctionBind
|
||||
),
|
||||
});
|
||||
}
|
||||
if envelope.version == FUNCTION_BINDINGS_VERSION
|
||||
&& envelope.bindings.iter().any(|binding| {
|
||||
binding
|
||||
.get("literal_inputs")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|inputs| !inputs.is_empty())
|
||||
})
|
||||
{
|
||||
return Err(invalid_function(
|
||||
"literal Function bindings require metadata version 2",
|
||||
));
|
||||
}
|
||||
Ok(Some(envelope))
|
||||
}
|
||||
|
||||
@@ -401,6 +432,7 @@ pub(crate) fn ensure_supported_function_metadata(schema: &ArrowSchema) -> Result
|
||||
.collect::<Result<Vec<FunctionBinding>>>()?;
|
||||
let mut binding_ids = BTreeSet::new();
|
||||
for binding in &bindings {
|
||||
binding.validate_inputs()?;
|
||||
if !binding_ids.insert(binding.binding_id().to_string()) {
|
||||
return Err(Error::InvalidInput {
|
||||
message: format!("duplicate Function binding '{}'", binding.binding_id()),
|
||||
@@ -821,6 +853,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
|
||||
"binding_id",
|
||||
"function",
|
||||
"inputs",
|
||||
"literal_inputs",
|
||||
"outputs",
|
||||
"assignment",
|
||||
"input_schema",
|
||||
@@ -830,6 +863,14 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
|
||||
"binding",
|
||||
)?;
|
||||
let object = value.as_object().unwrap();
|
||||
if let Some(inputs) = object.get("literal_inputs") {
|
||||
for input in inputs
|
||||
.as_array()
|
||||
.ok_or_else(|| invalid_function("Function literal_inputs must be an array"))?
|
||||
{
|
||||
reject_unknown_object_fields(input, &["parameter", "value"], "literal input binding")?;
|
||||
}
|
||||
}
|
||||
reject_unknown_object_fields(
|
||||
object
|
||||
.get("function")
|
||||
@@ -1305,6 +1346,7 @@ fn nullable_scalar_allows_assignment(output: &OutputMapping) -> Result<bool> {
|
||||
}
|
||||
|
||||
fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> {
|
||||
binding.validate_inputs()?;
|
||||
let mut input_fields = Vec::with_capacity(binding.inputs().len());
|
||||
for input in binding.inputs() {
|
||||
let resolved = resolve_field_path(schema, &input.field_path)?;
|
||||
@@ -1350,6 +1392,52 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
}
|
||||
input_fields.push(parameter_field);
|
||||
}
|
||||
if !binding.literal_inputs().is_empty() {
|
||||
// Column types still come from the table. Literal types come from the
|
||||
// complete signature schema persisted by the Enterprise service.
|
||||
let declared: JsonArrowSchema =
|
||||
serde_json::from_value(binding.input_schema().cloned().ok_or_else(|| {
|
||||
invalid_function("literal Function binding requires an input schema")
|
||||
})?)
|
||||
.map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?;
|
||||
let mut columns = input_fields
|
||||
.into_iter()
|
||||
.map(|field| (field.name().clone(), field))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut literals = binding
|
||||
.literal_inputs()
|
||||
.iter()
|
||||
.map(|input| (input.parameter.as_str(), &input.value))
|
||||
.collect::<HashMap<_, _>>();
|
||||
input_fields = Vec::with_capacity(declared.fields.len());
|
||||
for field in &declared.fields {
|
||||
if let Some(column) = columns.remove(&field.name) {
|
||||
input_fields.push(column);
|
||||
} else if let Some(value) = literals.remove(field.name.as_str()) {
|
||||
let field =
|
||||
lance_namespace::schema::convert_json_arrow_field(field).map_err(|e| {
|
||||
invalid_function(format!("invalid Function literal field: {e}"))
|
||||
})?;
|
||||
if field.is_blob_v2() || (value.is_null() && !field.is_nullable()) {
|
||||
return Err(invalid_function(format!(
|
||||
"invalid literal for Function parameter '{}'",
|
||||
field.name()
|
||||
)));
|
||||
}
|
||||
input_fields.push(field);
|
||||
} else {
|
||||
return Err(invalid_function(format!(
|
||||
"Function input schema has an unknown or duplicate parameter '{}'",
|
||||
field.name
|
||||
)));
|
||||
}
|
||||
}
|
||||
if !columns.is_empty() || !literals.is_empty() {
|
||||
return Err(invalid_function(
|
||||
"Function input schema is missing bound parameters",
|
||||
));
|
||||
}
|
||||
}
|
||||
let input_schema =
|
||||
lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(input_fields))
|
||||
.map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?;
|
||||
@@ -1502,6 +1590,10 @@ pub(crate) fn plan_function_application(
|
||||
input.parameter
|
||||
)));
|
||||
}
|
||||
if input.kind == "literal" {
|
||||
crate::function::validate_literal(&input.value)?;
|
||||
continue;
|
||||
}
|
||||
if input.kind != "column" {
|
||||
return Err(Error::NotSupported {
|
||||
message: format!(
|
||||
@@ -1680,7 +1772,15 @@ pub(crate) fn plan_function_application(
|
||||
|
||||
Ok(FunctionDeclarationPlan {
|
||||
application: application.clone(),
|
||||
binding_metadata_version: FUNCTION_BINDINGS_VERSION,
|
||||
binding_metadata_version: if application
|
||||
.inputs()
|
||||
.iter()
|
||||
.any(|input| input.kind == "literal")
|
||||
{
|
||||
FUNCTION_LITERAL_BINDINGS_VERSION
|
||||
} else {
|
||||
FUNCTION_BINDINGS_VERSION
|
||||
},
|
||||
input_bindings,
|
||||
input_schema,
|
||||
output_schema: JsonArrowSchema::new(output_fields),
|
||||
@@ -4738,6 +4838,129 @@ mod tests {
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(fields), &binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_literal_function_application_keeps_only_column_dependencies() {
|
||||
let mut value = serde_json::to_value(single_input_application("title")).unwrap();
|
||||
value["inputs"] = serde_json::json!([
|
||||
{"parameter": "count", "kind": "literal", "value": 16},
|
||||
{"parameter": "text", "kind": "column", "value": {"path": "title"}},
|
||||
{"parameter": "optional", "kind": "literal", "value": null}
|
||||
]);
|
||||
let application = FunctionApplication::from_json(&value.to_string()).unwrap();
|
||||
let plan =
|
||||
plan_function_application(&function_input_schema(), &application, Some("result"))
|
||||
.unwrap();
|
||||
assert_eq!(plan.binding_metadata_version, 2);
|
||||
assert_eq!(plan.input_bindings.len(), 1);
|
||||
assert_eq!(plan.input_bindings[0].parameter, "text");
|
||||
assert_eq!(plan.input_bindings[0].field_path, "title");
|
||||
assert_eq!(plan.input_schema.fields.len(), 1);
|
||||
assert_eq!(plan.input_schema.fields[0].name, "text");
|
||||
assert_eq!(
|
||||
serde_json::to_value(plan.application).unwrap()["inputs"],
|
||||
value["inputs"]
|
||||
);
|
||||
value["inputs"][0]["parameter"] = serde_json::json!("text");
|
||||
let duplicate = FunctionApplication::from_json(&value.to_string()).unwrap();
|
||||
assert!(
|
||||
plan_function_application(&function_input_schema(), &duplicate, Some("result"))
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("duplicate Function parameter")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_literal_function_metadata_survives_schema_mutation_guards() {
|
||||
let legacy = FunctionBinding::from_json(include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
|
||||
))
|
||||
.unwrap();
|
||||
let legacy_envelope: Value = serde_json::from_str(
|
||||
&function_bindings_metadata(std::slice::from_ref(&legacy)).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(legacy_envelope["version"], 1);
|
||||
let mut value = serde_json::to_value(&legacy).unwrap();
|
||||
value["inputs"].as_array_mut().unwrap().pop();
|
||||
value["literal_inputs"] = serde_json::json!([{"parameter": "body", "value": null}]);
|
||||
let binding = FunctionBinding::from_json(&value.to_string()).unwrap();
|
||||
let raw = function_bindings_metadata(std::slice::from_ref(&binding)).unwrap();
|
||||
let mut envelope: Value = serde_json::from_str(&raw).unwrap();
|
||||
assert_eq!(envelope["version"], 2);
|
||||
assert_eq!(
|
||||
envelope["bindings"][0]["literal_inputs"],
|
||||
value["literal_inputs"]
|
||||
);
|
||||
let schema = valid_function_binding_schema(true, true, &binding);
|
||||
let schema = schema.with_metadata(HashMap::from([(
|
||||
FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
raw.clone(),
|
||||
)]));
|
||||
ensure_supported_function_metadata(&schema).unwrap();
|
||||
let decoded = function_bindings(&schema).unwrap();
|
||||
assert_eq!(decoded.as_slice(), std::slice::from_ref(&binding));
|
||||
assert_eq!(function_bindings_metadata(&decoded).unwrap(), raw);
|
||||
assert!(!protected_roots(&decoded).unwrap().contains("body"));
|
||||
|
||||
envelope["version"] = serde_json::json!(1);
|
||||
let downgraded = schema.clone().with_metadata(HashMap::from([(
|
||||
FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
envelope.to_string(),
|
||||
)]));
|
||||
assert!(
|
||||
function_bindings(&downgraded)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("require metadata version 2")
|
||||
);
|
||||
|
||||
value["literal_inputs"][0]["parameter"] = serde_json::json!("title");
|
||||
assert!(
|
||||
FunctionBinding::from_json(&value.to_string())
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("duplicate Function parameter")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_literal_function_binding_requires_complete_input_schema() {
|
||||
let mut value: Value = serde_json::from_str(include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_function_binding.json"
|
||||
))
|
||||
.unwrap();
|
||||
value["inputs"].as_array_mut().unwrap().pop();
|
||||
value["literal_inputs"] = serde_json::json!([{"parameter": "body", "value": null}]);
|
||||
value["input_schema"]["fields"][1]["nullable"] = serde_json::json!(false);
|
||||
let binding = FunctionBinding::from_json(&value.to_string()).unwrap();
|
||||
let schema = valid_function_binding_schema(true, true, &binding);
|
||||
assert!(
|
||||
ensure_binding_matches_schema(&schema, &binding)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("invalid literal")
|
||||
);
|
||||
value["input_schema"]["fields"]
|
||||
.as_array_mut()
|
||||
.unwrap()
|
||||
.pop();
|
||||
let missing = FunctionBinding::from_json(&value.to_string()).unwrap();
|
||||
assert!(
|
||||
ensure_binding_matches_schema(&schema, &missing)
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("missing bound parameters")
|
||||
);
|
||||
value["literal_inputs"][0]["value"] = serde_json::json!(1.5);
|
||||
assert!(
|
||||
FunctionBinding::from_json(&value.to_string())
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("floating-point Function literals")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_function_binding_metadata_survives_schema_round_trip() {
|
||||
let binding = FunctionBinding::from_json(include_str!(
|
||||
@@ -5391,7 +5614,7 @@ mod tests {
|
||||
function_input_schema().fields().to_vec(),
|
||||
HashMap::from([(
|
||||
FUNCTION_BINDINGS_META_KEY.to_string(),
|
||||
r#"{"version":2,"bindings":[]}"#.to_string(),
|
||||
r#"{"version":3,"bindings":[]}"#.to_string(),
|
||||
)]),
|
||||
);
|
||||
let err = plan_function_application(&mixed_schema, &named_struct_application("{}"), None)
|
||||
|
||||
Reference in New Issue
Block a user