diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 36f47c8f0..5aa21f691 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -578,6 +578,8 @@ with BlobReader(read_range, len(payload)) as reader: ::: lancedb.functions.InputBinding +::: lancedb.functions.LiteralInputBinding + ::: lancedb.functions.OutputMapping ::: lancedb.functions.AssignmentMapping diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 57a16dbf2..41e64d482 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -245,6 +245,52 @@ function application keeps its scalar or named-struct result in the named column. A bare named-struct application expands its result fields as one atomic binding, with aliases from `rename(columns=...)`. +{/* Draft for the Function literal binding release. + +### Bind a constant input {#literal-inputs} + + +Literal inputs require an SDK that supports binding metadata version 2 and an +Enterprise deployment with matching literal declaration and execution support. +Upgrading the SDK alone does not enable this server capability. Older clients +reject version 2 bindings; column-only bindings continue to use version 1. + + +Use `lit(...)` to pass the same value for every row while another parameter +reads a column. For example, using the table above: + + +```python Python icon="python" +from lancedb import col, function, lit + + +@function +def multiply(value: float, factor: int) -> float: + return value * factor + + +multiplier = db.create_function_async(multiply).wait() +table.add_columns( + {"scaled": multiplier(value=col("value"), factor=lit(16))}, + auto_refresh_enabled=False, +) +``` + + +Supported scalar constants are booleans, strings, integers from `-(2**63)` +through `2**64 - 1`, and `None` for a nullable parameter. Each value must match +the parameter's declared type. Floating-point, binary, date, datetime and +decimal constants are not supported in Function input bindings, even though +`lit(...)` accepts them in other SQL expressions. Bind a direct literal; +expressions such as `lit(16) + lit(8)` are not supported here. + +The constant belongs to the column binding, so the Function version and its +image stay the same. Only column inputs contribute table dependencies. Refresh +`scaled` to fill it. To change the constant to `lit(8)`, drop and redeclare the +column using the same `multiplier` version; the new binding needs a refresh. + +*/} + ### Bind more than one input, or fill more than one column {#multi} Parameters bind by name, so a function that takes several arguments binds diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index cce3efeef..a87b82a21 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -117,7 +117,7 @@ class PyExpr: def expr_col(name: str) -> PyExpr: ... def expr_lit( - value: Union[bool, int, float, str, bytes, date, datetime, Decimal], + value: Union[bool, int, float, str, bytes, date, datetime, Decimal, None], ) -> PyExpr: ... def expr_func(name: str, args: List[PyExpr]) -> PyExpr: ... diff --git a/python/python/lancedb/expr.py b/python/python/lancedb/expr.py index f65fbd21a..b25e5d012 100644 --- a/python/python/lancedb/expr.py +++ b/python/python/lancedb/expr.py @@ -29,6 +29,8 @@ from lancedb._lancedb import PyExpr, expr_col, expr_lit, expr_func __all__ = ["Expr", "col", "lit", "func"] +_NOT_LITERAL = object() + _STR_TO_PA_TYPE: dict = { "bool": pa.bool_(), "boolean": pa.bool_(), @@ -92,9 +94,16 @@ class Expr: # for dict keys / set membership. __hash__ = None # type: ignore[assignment] - def __init__(self, inner: PyExpr, *, column_path: str | None = None) -> None: + def __init__( + self, + inner: PyExpr, + *, + column_path: str | None = None, + literal_value: object = _NOT_LITERAL, + ) -> None: self._inner = inner self._column_path = column_path + self._literal_value = literal_value def __bool__(self) -> bool: # An Expr describes a predicate that the query engine evaluates per row, @@ -301,14 +310,16 @@ def col(name: str) -> Expr: return Expr(expr_col(name), column_path=name) -def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> Expr: +def lit( + value: Union[bool, int, float, str, bytes, date, datetime, Decimal, None], +) -> Expr: """Create a literal (constant) value expression. Parameters ---------- value: A Python ``bool``, ``int``, ``float``, ``str``, ``bytes``, ``date``, - ``datetime``, or ``Decimal``. + ``datetime``, ``Decimal``, or ``None`` for SQL NULL. Examples -------- @@ -316,10 +327,12 @@ def lit(value: Union[bool, int, float, str, bytes, date, datetime, Decimal]) -> >>> col("price") * lit(1.1) Expr((price * 1.1)) """ - if not isinstance(value, (bool, int, float, str, bytes, date, datetime, Decimal)): + if value is not None and not isinstance( + value, (bool, int, float, str, bytes, date, datetime, Decimal) + ): raise TypeError(f"Unsupported literal type: {type(value).__name__}") - return Expr(expr_lit(value)) + return Expr(expr_lit(value), literal_value=value) def func(name: str, *args: ExprLike) -> Expr: diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 86230be52..471fc2437 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -158,11 +158,15 @@ def _validate_literal(value): "Function integer literal is outside the canonical JSON range" ) elif isinstance(value, Mapping): + if not all(isinstance(key, str) for key in value): + raise TypeError("Function literal object keys must be strings") for child in value.values(): _validate_literal(child) elif isinstance(value, (list, tuple)): for child in value: _validate_literal(child) + elif value is not None and not isinstance(value, (bool, str)): + raise TypeError(f"unsupported Function literal type: {type(value).__name__}") return value @@ -193,7 +197,7 @@ class _RemoteValue(BaseModel): known = {} for name, field in self.__class__.model_fields.items(): value = getattr(self, name) - if value is None: + if value is None and not field.is_required(): continue if not field.is_required(): default_factory = field.default_factory @@ -626,10 +630,17 @@ class FunctionVersion(_RemoteValue): return _validate_row_timeout(value) def __call__(self, **arguments: Any) -> FunctionApplication: - """Bind this exact version to named table columns. + """Bind this exact version to named table columns and constant inputs. Every input must be a direct [lancedb.col][lancedb.expr.col] - reference. A Function that declares initialization fields takes their + reference or [lancedb.lit][lancedb.expr.lit] constant. Constants support + integers, strings, booleans, and null; floating-point, binary and + temporal literals are not in the canonical Function input contract. + The Enterprise service must support literal bindings and validates + constants against this exact version's input signature. To change a + constant, redeclare the column using the same Function version. + + A Function that declares initialization fields takes their values as further keyword arguments; they are constants of this binding, fixed for every row, and every instance the binding runs is created with them. A non-nullable field without a default must be @@ -658,7 +669,7 @@ class FunctionVersion(_RemoteValue): ... "embedding": embed(text=col("body"), model="small", dimensions=512) ... }) """ - from lancedb.expr import Expr + from lancedb.expr import Expr, _NOT_LITERAL parameters = tuple(parameter.name for parameter in self.signature.inputs) fields = {field.name: field for field in self.signature.initialization} @@ -682,17 +693,27 @@ class FunctionVersion(_RemoteValue): bindings = [] for parameter in parameters: value = arguments[parameter] - if not isinstance(value, Expr) or value._column_path is None: + if isinstance(value, Expr) and value._literal_value is not _NOT_LITERAL: + bindings.append( + ApplicationInput( + parameter=parameter, + kind="literal", + value=value._literal_value, + ) + ) + elif isinstance(value, Expr) and value._column_path is not None: + bindings.append( + ApplicationInput( + parameter=parameter, + kind="column", + value={"path": value._column_path}, + ) + ) + else: raise TypeError( - f"Function input {parameter!r} must be a direct col(...) reference" + f"Function input {parameter!r} must be a direct col(...) reference " + "or lit(...) constant" ) - bindings.append( - ApplicationInput( - parameter=parameter, - kind="column", - value={"path": value._column_path}, - ) - ) initialization = {} for name in fields: if name in arguments: @@ -876,6 +897,18 @@ class InputBinding(_RemoteValue): nullable: bool +class LiteralInputBinding(_RemoteValue): + """Constant supplied to a Function parameter for every input row.""" + + parameter: str + value: Any + + @field_validator("value") + @classmethod + def _validate_value(cls, value): + return _validate_literal(value) + + class OutputMapping(_RemoteValue): """One stable result-field mapping.""" @@ -900,6 +933,7 @@ class FunctionBinding(_RemoteValue): binding_id: str function: FunctionVersionRef inputs: tuple[InputBinding, ...] + literal_inputs: tuple[LiteralInputBinding, ...] = () outputs: tuple[OutputMapping, ...] assignment: Optional[AssignmentMapping] = None input_schema: Optional[Mapping[str, Any]] = None @@ -907,6 +941,13 @@ class FunctionBinding(_RemoteValue): initialization: Optional[str] = None """Standard base64 of the validated one-row Arrow IPC initialization stream.""" + @model_validator(mode="after") + def _validate_input_parameters(self): + parameters = [input.parameter for input in (*self.inputs, *self.literal_inputs)] + if len(parameters) != len(set(parameters)): + raise ValueError("Function binding input parameters must be unique") + return self + def initialization_row(self) -> Optional[dict[str, Any]]: """The initialization values this binding creates instances with.""" if self.initialization is None: @@ -2979,6 +3020,7 @@ __all__ = [ "FunctionVersionRef", "HuggingFaceModel", "InputBinding", + "LiteralInputBinding", "OutputMapping", "PythonEnvironmentSpec", "PythonAdapterSpec", diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index 867672fe9..07be57e15 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -8,7 +8,7 @@ from types import MappingProxyType import pytest -from lancedb import col +from lancedb import col, lit import lancedb.functions as functions from lancedb.functions import ( FunctionApplication, @@ -58,6 +58,7 @@ def assert_no_secret_values(value): def test_public_function_values_are_in_api_reference(): + assert "LiteralInputBinding" in functions.__all__ docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md" rendered = docs.read_text() for name in functions.__all__: @@ -286,6 +287,67 @@ def test_function_version_binding_validates_names_and_direct_columns(): version(text=col("text").lower()) +@pytest.mark.parametrize("constant", [16, 8, True, "prompt", None, 2**64 - 1]) +def test_function_version_binds_direct_literals_without_changing_identity(constant): + value = job_result("remote_function_job.json") + value["signature"]["inputs"].append( + { + "name": "option", + "arrow_type": "bool" + if type(constant) is bool + else "utf8" + if isinstance(constant, str) + else "uint64" + if constant == 2**64 - 1 + else "int64", + "nullable": True, + } + ) + version = FunctionVersion.from_json(json.dumps(value)) + expression = lit(constant) + application = version(text=col("body"), option=expression) + encoded = json.loads(application.to_canonical_json()) + assert encoded["inputs"] == [ + {"parameter": "text", "kind": "column", "value": {"path": "body"}}, + {"parameter": "option", "kind": "literal", "value": constant}, + ] + assert application.function.version == version.version + assert application.function.object_id == version.object_id + assert application.function.manifest_digest == version.image.manifest_digest + assert FunctionApplication.from_json(application.to_canonical_json()) == application + if constant is None: + assert expression.to_sql() == "NULL" + if type(constant) is int: + assert expression.to_sql() == str(constant) + + +def test_function_literals_reject_expressions_and_unsupported_values(): + version = FunctionVersion.from_json( + json.dumps(job_result("remote_function_job.json")) + ) + with pytest.raises(TypeError, match="direct col"): + version(text=lit(16) + lit(8)) + with pytest.raises(ValueError, match="floating-point Function literals"): + version(text=lit(1.5)) + with pytest.raises(TypeError, match="unsupported Function literal type"): + version(text=lit(b"data")) + + +def test_function_literal_binding_preserves_null_and_rejects_duplicate_parameters(): + value = json.loads((FIXTURES / "remote_function_binding.json").read_text()) + value["inputs"] = value["inputs"][:1] + value["literal_inputs"] = [{"parameter": "body", "value": None}] + binding = FunctionBinding.from_json(json.dumps(value)) + assert binding.literal_inputs[0].value is None + assert json.loads(binding.to_canonical_json())["literal_inputs"] == [ + {"parameter": "body", "value": None} + ] + assert FunctionBinding.from_json(binding.to_canonical_json()) == binding + value["literal_inputs"][0]["parameter"] = "title" + with pytest.raises(ValueError, match="input parameters must be unique"): + FunctionBinding.from_json(json.dumps(value)) + + def test_function_version_keeps_named_struct_outputs_in_one_application(): value = job_result("remote_function_job.json") value["name"] = "text_features" diff --git a/python/src/expr.rs b/python/src/expr.rs index bb779bc1d..c4c7cbbb1 100644 --- a/python/src/expr.rs +++ b/python/src/expr.rs @@ -164,9 +164,12 @@ pub fn expr_col(name: &str) -> PyExpr { /// Create a literal value expression. /// /// Supported Python types: `bool`, `int`, `float`, `str`, `bytes`, `date`, -/// `datetime`, `Decimal`. +/// `datetime`, `Decimal`, and `None` (SQL NULL). #[pyfunction] pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(PyExpr(df_lit(ScalarValue::Null))); + } // bool must be checked before int because bool is a subclass of int in Python if let Ok(b) = value.extract::() { return Ok(PyExpr(df_lit(b))); @@ -174,6 +177,9 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult { if let Ok(i) = value.extract::() { return Ok(PyExpr(df_lit(i))); } + if let Ok(i) = value.extract::() { + return Ok(PyExpr(df_lit(i))); + } // Decimal must be checked before f64: Python's Decimal implements __float__, // so value.extract::() would succeed and silently truncate the value to // f64, losing precision. Build a Decimal128 scalar to preserve it instead. diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 387c80364..582b5b08a 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -111,7 +111,7 @@ fn from_json(json: &str) -> Result { serde_json::from_str(json).map_err(invalid_json) } -fn validate_literal(value: &Value) -> Result<()> { +pub(crate) fn validate_literal(value: &Value) -> Result<()> { match value { Value::Number(number) if number.is_f64() => Err(Error::InvalidInput { message: "floating-point Function literals are not part of the Slice 1 canonical wire contract" @@ -1018,6 +1018,18 @@ pub struct InputBinding { pub nullable: bool, } +/// Constant value bound to a Function parameter for every input row. +/// +/// Values use the same canonical JSON domain as [`ApplicationInput`]. The +/// Enterprise service validates their types against the exact Function version. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LiteralInputBinding { + /// Function parameter name, distinct from all column-bound parameters. + pub parameter: String, + /// Immutable canonical JSON value. + pub value: Value, +} + /// Ordered result-field to table-field mapping for a Function binding. /// /// `nullable` describes the logical Function result. Physical computed-column @@ -1046,6 +1058,8 @@ pub struct FunctionBinding { binding_id: String, function: FunctionVersionRef, inputs: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + literal_inputs: Vec, outputs: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] assignment: Option, @@ -1076,6 +1090,30 @@ impl FunctionBinding { &self.inputs } + /// Constants supplied for every row; these are not table dependencies. + pub fn literal_inputs(&self) -> &[LiteralInputBinding] { + &self.literal_inputs + } + + pub(crate) fn validate_inputs(&self) -> Result<()> { + let mut parameters = std::collections::HashSet::new(); + for parameter in self + .inputs + .iter() + .map(|input| &input.parameter) + .chain(self.literal_inputs.iter().map(|input| &input.parameter)) + { + if !parameters.insert(parameter) { + return Err(invalid_json(format!( + "duplicate Function parameter '{parameter}'" + ))); + } + } + self.literal_inputs + .iter() + .try_for_each(|input| validate_literal(&input.value)) + } + pub fn outputs(&self) -> &[OutputMapping] { &self.outputs } @@ -1098,7 +1136,20 @@ impl FunctionBinding { } } -impl_json!(FunctionBinding); +impl FunctionBinding { + /// Decode an immutable binding, validating its canonical input values. + pub fn from_json(json: &str) -> Result { + let binding: Self = from_json(json)?; + binding.validate_inputs()?; + Ok(binding) + } + + /// Encode this binding without dropping its constant inputs. + pub fn to_canonical_json(&self) -> Result { + self.validate_inputs()?; + canonical_json(self) + } +} /// Stable terminal result of an expression-backed or Function-backed column /// refresh [`crate::Job`]. diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index e556d131c..29f3ab50e 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -10769,9 +10769,17 @@ mod tests { } } + #[rstest] + #[case::column_only(None)] + #[case::sixteen(Some(serde_json::json!(16)))] + #[case::eight(Some(serde_json::json!(8)))] + #[case::null(Some(serde_json::Value::Null))] #[tokio::test] - async fn test_add_scalar_function_column_sends_atomic_null_declaration() { - let table = Table::new_with_handler("my_table", |request| { + async fn test_add_scalar_function_column_sends_atomic_null_declaration( + #[case] literal: Option, + ) { + let expected_literal = literal.clone(); + let table = Table::new_with_handler("my_table", move |request| { match request.url().path() { "/v1/table/my_table/describe/" => http::Response::builder() .status(200) @@ -10789,6 +10797,13 @@ mod tests { )) .unwrap(); expected["auto_refresh_enabled"] = serde_json::json!(true); + if let Some(literal) = &expected_literal { + expected["function"]["binding_metadata_version"] = serde_json::json!(2); + expected["function"]["application"]["inputs"] + .as_array_mut().unwrap().push(serde_json::json!({ + "parameter": "num_frames", "kind": "literal", "value": literal, + })); + } assert_eq!(actual, expected); http::Response::builder() .status(200) @@ -10798,7 +10813,7 @@ mod tests { path => panic!("Unexpected path: {path}"), } }); - let application = crate::function::FunctionApplication::from_json( + let mut application: serde_json::Value = serde_json::from_str( r#"{ "function":{"name":"embed","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], @@ -10806,6 +10821,18 @@ mod tests { }"#, ) .unwrap(); + if let Some(literal) = literal { + application["inputs"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "parameter": "num_frames", "kind": "literal", "value": literal, + })); + } + let application = crate::function::FunctionApplication::from_json( + &serde_json::to_string(&application).unwrap(), + ) + .unwrap(); let result = table .add_columns() @@ -10934,19 +10961,64 @@ mod tests { /// Only a column the binding uses is refused, and it is refused before /// any request goes out; the rest reach the server as usual. + #[rstest] + #[case::column_only(false)] + #[case::literal(true)] #[tokio::test] - async fn test_add_columns_scopes_to_the_columns_a_binding_uses() { - let table = Table::new_with_handler("my_table", |request| match request.url().path() { - "/v1/table/my_table/describe/" => http::Response::builder() - .status(200) - .body(describe_response(&fixture_bound_schema())) - .unwrap(), - "/v1/table/my_table/add_columns/" => http::Response::builder() - .status(200) - .body(r#"{"version":10}"#.to_string()) - .unwrap(), - path => panic!("Unexpected path: {path}"), - }); + async fn test_add_columns_scopes_to_the_columns_a_binding_uses(#[case] literal: bool) { + let mut schema = fixture_bound_schema(); + if literal { + let key = crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY; + let mut metadata = schema.metadata().clone(); + let mut envelope: serde_json::Value = + serde_json::from_str(metadata.get(key).unwrap()).unwrap(); + envelope["version"] = serde_json::json!(2); + envelope["bindings"][0]["inputs"] + .as_array_mut() + .unwrap() + .pop(); + envelope["bindings"][0]["literal_inputs"] = + serde_json::json!([{"parameter": "body", "value": null}]); + let binding = + crate::function::FunctionBinding::from_json(&envelope["bindings"][0].to_string()) + .unwrap(); + let fields = schema + .fields() + .iter() + .map(|field| { + let mut field = field.as_ref().clone(); + if let Some(output) = binding + .outputs() + .iter() + .find(|output| output.output_name == *field.name()) + { + field = field.with_metadata( + crate::table::computed_columns::function_computed_column_metadata( + binding.binding_id(), + output.output_ordinal, + &["title".into()], + ), + ); + } + field + }) + .collect::>(); + metadata.insert(key.to_string(), envelope.to_string()); + schema = Schema::new_with_metadata(fields, metadata); + } + let allowed_schema = schema.clone(); + let table = + Table::new_with_handler("my_table", move |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&allowed_schema)) + .unwrap(), + "/v1/table/my_table/add_columns/" => http::Response::builder() + .status(200) + .body(r#"{"version":10}"#.to_string()) + .unwrap(), + path => panic!("Unexpected path: {path}"), + }); table .add_columns() .computed("doubled", "spare * 2") @@ -10963,13 +11035,14 @@ mod tests { .await .unwrap(); - let table = Table::new_with_handler("my_table", |request| match request.url().path() { - "/v1/table/my_table/describe/" => http::Response::builder() - .status(200) - .body(describe_response(&fixture_bound_schema())) - .unwrap(), - path => panic!("mutation request must not be sent: {path}"), - }); + let table = + Table::new_with_handler("my_table", move |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&schema)) + .unwrap(), + path => panic!("mutation request must not be sent: {path}"), + }); for name in ["title", "search_text"] { let err = table .add_columns() @@ -11327,7 +11400,7 @@ mod tests { "/v1/table/my_table/describe/" => http::Response::builder() .status(200) .body( - r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":2,\"bindings\":[]}"}}}"#, + r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":3,\"bindings\":[]}"}}}"#, ) .unwrap(), path => panic!("mutation request must not be sent: {path}"), diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index b01f45d06..7348e659d 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -203,6 +203,10 @@ pub const FUNCTION_BINDINGS_META_KEY: &str = "lancedb::function_bindings"; /// Version of the schema-level Function binding envelope. pub const FUNCTION_BINDINGS_VERSION: u32 = 1; +/// Envelope version required when a Function binding contains constant inputs. +/// Older readers reject this version instead of dropping those inputs. +pub const FUNCTION_LITERAL_BINDINGS_VERSION: u32 = 2; + /// Field metadata key holding `{fragment id -> input signature}` as JSON, /// recorded by the refresh that last computed each fragment. Outside the /// declaration namespace on purpose: a declaration is immutable through @@ -329,6 +333,17 @@ struct FunctionBindingEnvelope { /// Encode immutable Function bindings for schema-level persistence. pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result { + bindings + .iter() + .try_for_each(FunctionBinding::validate_inputs)?; + let version = if bindings + .iter() + .any(|binding| !binding.literal_inputs().is_empty()) + { + FUNCTION_LITERAL_BINDINGS_VERSION + } else { + FUNCTION_BINDINGS_VERSION + }; let bindings = bindings .iter() .map(serde_json::to_value) @@ -336,12 +351,10 @@ pub fn function_bindings_metadata(bindings: &[FunctionBinding]) -> Result Result> { .bindings .into_iter() .map(|binding| { - serde_json::from_value(binding).map_err(|e| Error::InvalidInput { - message: format!("invalid Function binding metadata: {e}"), - }) + let binding: FunctionBinding = + serde_json::from_value(binding).map_err(|e| Error::InvalidInput { + message: format!("invalid Function binding metadata: {e}"), + })?; + binding.validate_inputs()?; + Ok(binding) }) .collect() } @@ -370,7 +386,10 @@ fn function_binding_envelope(schema: &ArrowSchema) -> Result Result Result .collect::>>()?; let mut binding_ids = BTreeSet::new(); for binding in &bindings { + binding.validate_inputs()?; if !binding_ids.insert(binding.binding_id().to_string()) { return Err(Error::InvalidInput { message: format!("duplicate Function binding '{}'", binding.binding_id()), @@ -821,6 +853,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> { "binding_id", "function", "inputs", + "literal_inputs", "outputs", "assignment", "input_schema", @@ -830,6 +863,14 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> { "binding", )?; let object = value.as_object().unwrap(); + if let Some(inputs) = object.get("literal_inputs") { + for input in inputs + .as_array() + .ok_or_else(|| invalid_function("Function literal_inputs must be an array"))? + { + reject_unknown_object_fields(input, &["parameter", "value"], "literal input binding")?; + } + } reject_unknown_object_fields( object .get("function") @@ -1305,6 +1346,7 @@ fn nullable_scalar_allows_assignment(output: &OutputMapping) -> Result { } fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> { + binding.validate_inputs()?; let mut input_fields = Vec::with_capacity(binding.inputs().len()); for input in binding.inputs() { let resolved = resolve_field_path(schema, &input.field_path)?; @@ -1350,6 +1392,52 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding } input_fields.push(parameter_field); } + if !binding.literal_inputs().is_empty() { + // Column types still come from the table. Literal types come from the + // complete signature schema persisted by the Enterprise service. + let declared: JsonArrowSchema = + serde_json::from_value(binding.input_schema().cloned().ok_or_else(|| { + invalid_function("literal Function binding requires an input schema") + })?) + .map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?; + let mut columns = input_fields + .into_iter() + .map(|field| (field.name().clone(), field)) + .collect::>(); + let mut literals = binding + .literal_inputs() + .iter() + .map(|input| (input.parameter.as_str(), &input.value)) + .collect::>(); + input_fields = Vec::with_capacity(declared.fields.len()); + for field in &declared.fields { + if let Some(column) = columns.remove(&field.name) { + input_fields.push(column); + } else if let Some(value) = literals.remove(field.name.as_str()) { + let field = + lance_namespace::schema::convert_json_arrow_field(field).map_err(|e| { + invalid_function(format!("invalid Function literal field: {e}")) + })?; + if field.is_blob_v2() || (value.is_null() && !field.is_nullable()) { + return Err(invalid_function(format!( + "invalid literal for Function parameter '{}'", + field.name() + ))); + } + input_fields.push(field); + } else { + return Err(invalid_function(format!( + "Function input schema has an unknown or duplicate parameter '{}'", + field.name + ))); + } + } + if !columns.is_empty() || !literals.is_empty() { + return Err(invalid_function( + "Function input schema is missing bound parameters", + )); + } + } let input_schema = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(input_fields)) .map_err(|e| invalid_function(format!("invalid Function input schema: {e}")))?; @@ -1502,6 +1590,10 @@ pub(crate) fn plan_function_application( input.parameter ))); } + if input.kind == "literal" { + crate::function::validate_literal(&input.value)?; + continue; + } if input.kind != "column" { return Err(Error::NotSupported { message: format!( @@ -1680,7 +1772,15 @@ pub(crate) fn plan_function_application( Ok(FunctionDeclarationPlan { application: application.clone(), - binding_metadata_version: FUNCTION_BINDINGS_VERSION, + binding_metadata_version: if application + .inputs() + .iter() + .any(|input| input.kind == "literal") + { + FUNCTION_LITERAL_BINDINGS_VERSION + } else { + FUNCTION_BINDINGS_VERSION + }, input_bindings, input_schema, output_schema: JsonArrowSchema::new(output_fields), @@ -4738,6 +4838,129 @@ mod tests { ensure_binding_matches_schema(&ArrowSchema::new(fields), &binding).unwrap(); } + #[test] + fn test_literal_function_application_keeps_only_column_dependencies() { + let mut value = serde_json::to_value(single_input_application("title")).unwrap(); + value["inputs"] = serde_json::json!([ + {"parameter": "count", "kind": "literal", "value": 16}, + {"parameter": "text", "kind": "column", "value": {"path": "title"}}, + {"parameter": "optional", "kind": "literal", "value": null} + ]); + let application = FunctionApplication::from_json(&value.to_string()).unwrap(); + let plan = + plan_function_application(&function_input_schema(), &application, Some("result")) + .unwrap(); + assert_eq!(plan.binding_metadata_version, 2); + assert_eq!(plan.input_bindings.len(), 1); + assert_eq!(plan.input_bindings[0].parameter, "text"); + assert_eq!(plan.input_bindings[0].field_path, "title"); + assert_eq!(plan.input_schema.fields.len(), 1); + assert_eq!(plan.input_schema.fields[0].name, "text"); + assert_eq!( + serde_json::to_value(plan.application).unwrap()["inputs"], + value["inputs"] + ); + value["inputs"][0]["parameter"] = serde_json::json!("text"); + let duplicate = FunctionApplication::from_json(&value.to_string()).unwrap(); + assert!( + plan_function_application(&function_input_schema(), &duplicate, Some("result")) + .unwrap_err() + .to_string() + .contains("duplicate Function parameter") + ); + } + + #[test] + fn test_literal_function_metadata_survives_schema_mutation_guards() { + let legacy = FunctionBinding::from_json(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + )) + .unwrap(); + let legacy_envelope: Value = serde_json::from_str( + &function_bindings_metadata(std::slice::from_ref(&legacy)).unwrap(), + ) + .unwrap(); + assert_eq!(legacy_envelope["version"], 1); + let mut value = serde_json::to_value(&legacy).unwrap(); + value["inputs"].as_array_mut().unwrap().pop(); + value["literal_inputs"] = serde_json::json!([{"parameter": "body", "value": null}]); + let binding = FunctionBinding::from_json(&value.to_string()).unwrap(); + let raw = function_bindings_metadata(std::slice::from_ref(&binding)).unwrap(); + let mut envelope: Value = serde_json::from_str(&raw).unwrap(); + assert_eq!(envelope["version"], 2); + assert_eq!( + envelope["bindings"][0]["literal_inputs"], + value["literal_inputs"] + ); + let schema = valid_function_binding_schema(true, true, &binding); + let schema = schema.with_metadata(HashMap::from([( + FUNCTION_BINDINGS_META_KEY.to_string(), + raw.clone(), + )])); + ensure_supported_function_metadata(&schema).unwrap(); + let decoded = function_bindings(&schema).unwrap(); + assert_eq!(decoded.as_slice(), std::slice::from_ref(&binding)); + assert_eq!(function_bindings_metadata(&decoded).unwrap(), raw); + assert!(!protected_roots(&decoded).unwrap().contains("body")); + + envelope["version"] = serde_json::json!(1); + let downgraded = schema.clone().with_metadata(HashMap::from([( + FUNCTION_BINDINGS_META_KEY.to_string(), + envelope.to_string(), + )])); + assert!( + function_bindings(&downgraded) + .unwrap_err() + .to_string() + .contains("require metadata version 2") + ); + + value["literal_inputs"][0]["parameter"] = serde_json::json!("title"); + assert!( + FunctionBinding::from_json(&value.to_string()) + .unwrap_err() + .to_string() + .contains("duplicate Function parameter") + ); + } + + #[test] + fn test_literal_function_binding_requires_complete_input_schema() { + let mut value: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + )) + .unwrap(); + value["inputs"].as_array_mut().unwrap().pop(); + value["literal_inputs"] = serde_json::json!([{"parameter": "body", "value": null}]); + value["input_schema"]["fields"][1]["nullable"] = serde_json::json!(false); + let binding = FunctionBinding::from_json(&value.to_string()).unwrap(); + let schema = valid_function_binding_schema(true, true, &binding); + assert!( + ensure_binding_matches_schema(&schema, &binding) + .unwrap_err() + .to_string() + .contains("invalid literal") + ); + value["input_schema"]["fields"] + .as_array_mut() + .unwrap() + .pop(); + let missing = FunctionBinding::from_json(&value.to_string()).unwrap(); + assert!( + ensure_binding_matches_schema(&schema, &missing) + .unwrap_err() + .to_string() + .contains("missing bound parameters") + ); + value["literal_inputs"][0]["value"] = serde_json::json!(1.5); + assert!( + FunctionBinding::from_json(&value.to_string()) + .unwrap_err() + .to_string() + .contains("floating-point Function literals") + ); + } + #[test] fn test_function_binding_metadata_survives_schema_round_trip() { let binding = FunctionBinding::from_json(include_str!( @@ -5391,7 +5614,7 @@ mod tests { function_input_schema().fields().to_vec(), HashMap::from([( FUNCTION_BINDINGS_META_KEY.to_string(), - r#"{"version":2,"bindings":[]}"#.to_string(), + r#"{"version":3,"bindings":[]}"#.to_string(), )]), ); let err = plan_function_application(&mixed_schema, &named_struct_application("{}"), None)