From 3728d41f027892ee45c4ec94c01ff0622a0ca106 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 23 Sep 2026 23:31:49 +0800 Subject: [PATCH] feat: carry Function initialization in the canonical model (#4253) A Function instance is created once with an initialization row (`create(initialization, context)` in the Function Format), but nothing in the client model could say what that row holds or where its values come from, so every source-built Function ran with an empty row and users pushed configuration through `env=` strings instead (ENT-2516). This adds initialization to the canonical wire model without changing a Function's identity: - `FunctionSignature.initialization` lists the fields of the row, in order. It is omitted when empty, so existing signatures and FunctionVersion hashes are unchanged. - `FunctionApplication.initialization` carries a binding's values as JSON. The service validates them against the Function's fields and re-encodes them as Arrow, so this JSON is transport only and never hashed; that is why floats are accepted here while column-input literals keep the Slice 1 domain. - `FunctionBinding.initialization` is the validated one-row Arrow IPC stream (base64) that every instance of the binding is created with. Values belong to the binding, so one Function version can back several columns with different values. - The binding-shape check that guards schema mutations accepts the new field; without it a table holding an initialized binding refused further declarations. Rust and Python share new canonical goldens for an initialized application and binding. The authoring side (class `@udf`, initialization arguments, shipped helper code) is the stacked PR on top of this one. --- python/python/lancedb/functions.py | 21 +++++ .../tests/test_first_class_function_slice1.py | 41 +++++++++ rust/lancedb/src/function.rs | 87 +++++++++++++++++++ rust/lancedb/src/table/computed_columns.rs | 12 +++ .../tests/first_class_function_slice1.rs | 23 +++++ ...alized_function_application.canonical.json | 1 + ...mote_initialized_function_application.json | 37 ++++++++ ...nitialized_function_binding.canonical.json | 1 + .../remote_initialized_function_binding.json | 61 +++++++++++++ 9 files changed, 284 insertions(+) create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.canonical.json create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.json create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.canonical.json create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 8c6864cc3..21c8a31e0 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -256,8 +256,16 @@ class SecretBinding(_RemoteValue): class FunctionSignature(_RemoteValue): + """Ordered inputs, output, and initialization fields of a Function. + + ``initialization`` lists the fields of the single row each Function + instance is created with. Values come from each binding, not from the + Function version; the tuple is empty when the Function takes none. + """ + inputs: tuple[FunctionParameter, ...] output: FunctionOutput + initialization: tuple[FunctionParameter, ...] = () class PythonEnvironmentSpec(_RemoteValue): @@ -478,6 +486,10 @@ class FunctionApplication(_OpenRemoteValue): inputs: tuple[ApplicationInput, ...] output: FunctionOutput columns: Mapping[str, str] = Field(default_factory=dict) + # Transport JSON: the service validates it against the Function's + # initialization schema and persists the Arrow row in the binding, so it is + # never hashed and floating-point values are allowed. + initialization: Mapping[str, Any] = Field(default_factory=dict) def _known_dict(self) -> dict[str, Any]: value = super()._known_dict() @@ -560,6 +572,15 @@ class FunctionBinding(_RemoteValue): assignment: Optional[AssignmentMapping] = None input_schema: Optional[Mapping[str, Any]] = None output_schema: Optional[Mapping[str, Any]] = None + initialization: Optional[str] = None + """Standard base64 of the validated one-row Arrow IPC initialization stream.""" + + def initialization_row(self) -> Optional[dict[str, Any]]: + """The initialization values this binding creates instances with.""" + if self.initialization is None: + return None + reader = pa.ipc.open_stream(base64.b64decode(self.initialization)) + return reader.read_all().to_pylist()[0] class RefreshColumnResult(_RemoteValue): diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index fbcfa03b1..849923fda 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -83,6 +83,18 @@ def test_public_function_values_are_in_api_reference(): FunctionBinding, False, ), + ( + "remote_initialized_function_application.json", + "remote_initialized_function_application.canonical.json", + FunctionApplication, + False, + ), + ( + "remote_initialized_function_binding.json", + "remote_initialized_function_binding.canonical.json", + FunctionBinding, + False, + ), ( "remote_refresh_job.json", "remote_refresh_result.canonical.json", @@ -269,6 +281,35 @@ def test_binding_and_refresh_result_keep_stable_remote_fields(): assert RefreshColumnResult.from_json(result.to_canonical_json()) == result +def test_initialization_travels_on_the_application_and_binding(): + application = FunctionApplication.from_json( + fixture("remote_initialized_function_application.json") + ) + assert application.initialization["temperature"] == 0.25 + assert application.initialization["retry"]["backoff"] == (1, 2, 4) + application._ensure_declarable() + + binding = FunctionBinding.from_json( + fixture("remote_initialized_function_binding.json") + ) + assert binding.initialization_row() == { + "model": "text-embedding-3-small", + "dimensions": 512, + } + assert ( + FunctionBinding.from_json( + fixture("remote_function_binding.json") + ).initialization_row() + is None + ) + + signature = FunctionVersion.from_json( + json.dumps(job_result("remote_function_job.json")) + ).signature + assert signature.initialization == () + assert "initialization" not in json.loads(signature.to_canonical_json()) + + def test_function_literal_numeric_domain_matches_rust(): with pytest.raises(ValueError, match="floating-point Function literals"): FunctionApplication.from_json(fixture("remote_function_application_float.json")) diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 3f790b6ab..3b7209bd5 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -187,6 +187,11 @@ pub struct FunctionOutput { pub struct FunctionSignature { pub inputs: Vec, pub output: FunctionOutput, + /// Ordered fields of the single initialization row a Function instance is + /// created with. Each binding supplies its own values; they are not part + /// of the Function version. Empty when the Function takes none. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub initialization: Vec, } /// One Python environment source. @@ -579,6 +584,13 @@ pub struct FunctionApplication { output: FunctionOutput, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] columns: BTreeMap, + /// Initialization values for this binding, by initialization field name. + /// The service validates them against the Function's initialization + /// schema and persists the resulting Arrow row in the binding; this JSON + /// is transport only and never hashed into an identity, so floating-point + /// values are accepted here. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + initialization: BTreeMap, #[serde(default, flatten, skip_serializing)] unknown_fields: BTreeMap, #[serde(default, skip)] @@ -601,6 +613,11 @@ impl FunctionApplication { pub fn columns(&self) -> &BTreeMap { &self.columns } + + /// Initialization values supplied by this application, by field name. + pub fn initialization(&self) -> &BTreeMap { + &self.initialization + } /// Whether a newer writer attached application fields this client cannot /// validate. Such applications remain readable but must not be declared. pub fn has_unknown_fields(&self) -> bool { @@ -677,6 +694,11 @@ pub struct FunctionBinding { /// Exact physical Arrow schema of the binding's table outputs. #[serde(default, skip_serializing_if = "Option::is_none")] output_schema: Option, + /// Standard base64 of the one-row Arrow IPC stream every instance of this + /// binding is created with, already validated against the Function's + /// initialization schema. Absent when the Function takes none. + #[serde(default, skip_serializing_if = "Option::is_none")] + initialization: Option, } impl FunctionBinding { @@ -707,6 +729,11 @@ impl FunctionBinding { pub fn output_schema(&self) -> Option<&Value> { self.output_schema.as_ref() } + + /// Base64 one-row Arrow IPC initialization stream, if the Function takes one. + pub fn initialization(&self) -> Option<&str> { + self.initialization.as_deref() + } } impl_json!(FunctionBinding); @@ -964,6 +991,66 @@ mod tests { } } + /// A Function without initialization keeps its pre-initialization wire + /// form, so existing signatures and applications encode byte-identically. + #[test] + fn empty_initialization_is_omitted_from_the_wire() { + let signature: FunctionSignature = serde_json::from_str( + r#"{"inputs":[{"name":"x","arrow_type":"int64","nullable":true}],"output":{"kind":"scalar","arrow_type":"int64","nullable":false}}"#, + ) + .unwrap(); + assert!(signature.initialization.is_empty()); + assert!( + !canonical_json(&signature) + .unwrap() + .contains("initialization") + ); + + let signature: FunctionSignature = serde_json::from_str( + r#"{"inputs":[],"output":{"kind":"scalar","arrow_type":"int64","nullable":false},"initialization":[{"name":"factor","arrow_type":"float64","nullable":false}]}"#, + ) + .unwrap(); + assert_eq!(signature.initialization[0].name, "factor"); + assert!( + canonical_json(&signature) + .unwrap() + .contains(r#""initialization":[{"#) + ); + } + + /// Initialization values are transport JSON, validated and re-encoded as + /// Arrow by the service, so floats are accepted there while column-input + /// literals keep the Slice 1 domain. + #[test] + fn application_initialization_accepts_floats() { + let application = FunctionApplication::from_json( + r#"{"function":{"name":"scale","object_id":"o","location":"l","version":"1","manifest_digest":"sha256:x"},"inputs":[{"parameter":"x","kind":"column","value":{"path":"x"}}],"output":{"kind":"scalar","arrow_type":"float64","nullable":false},"initialization":{"factor":2.5,"label":"fast"}}"#, + ) + .unwrap(); + assert!(!application.has_unknown_fields()); + assert_eq!( + application.initialization()["factor"], + serde_json::json!(2.5) + ); + let encoded = application.to_canonical_json().unwrap(); + assert!(encoded.contains(r#""initialization":{"factor":2.5,"label":"fast"}"#)); + } + + #[test] + fn binding_initialization_round_trips() { + let binding = FunctionBinding::from_json( + r#"{"binding_id":"fb_1","function":{"name":"scale","object_id":"o","location":"l","version":"1","manifest_digest":"sha256:x"},"inputs":[],"outputs":[],"initialization":"QUJD"}"#, + ) + .unwrap(); + assert_eq!(binding.initialization(), Some("QUJD")); + assert!( + binding + .to_canonical_json() + .unwrap() + .ends_with(r#""initialization":"QUJD","inputs":[],"outputs":[]}"#) + ); + } + /// Unknown keys are how a newer server's payload reaches an older client, /// so the check has to be exact about which level it is looking at. #[test] diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index 679928cd3..98b4e1627 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -695,6 +695,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> { "assignment", "input_schema", "output_schema", + "initialization", ], "binding", )?; @@ -3748,6 +3749,17 @@ mod tests { assert!(matches!(err, Error::NotSupported { .. })); } + /// A binding's initialization row is part of the known contract, so a + /// table that holds one can still take further declarations. + #[test] + fn test_initialized_bindings_are_a_known_shape() { + let raw_binding: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json" + )) + .unwrap(); + ensure_known_binding_shape(&raw_binding).unwrap(); + } + #[test] fn test_named_struct_can_be_kept_as_one_nullable_physical_column() { let application = named_struct_application("{}"); diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index 3971a2471..86ccea1cc 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -115,6 +115,29 @@ fn application_and_binding_match_shared_remote_goldens() { ); } +/// Initialization values travel as JSON on the application, including +/// floats, and as the validated Arrow row on the binding. +#[test] +fn initialized_application_and_binding_match_shared_remote_goldens() { + let application = + FunctionApplication::from_json(&fixture("remote_initialized_function_application.json")) + .expect("application fixture"); + assert!(!application.has_unknown_fields()); + assert_eq!(application.initialization()["temperature"], 0.25); + assert_eq!( + application.to_canonical_json().expect("canonical JSON"), + fixture("remote_initialized_function_application.canonical.json").trim() + ); + + let binding = FunctionBinding::from_json(&fixture("remote_initialized_function_binding.json")) + .expect("binding fixture"); + assert!(binding.initialization().is_some()); + assert_eq!( + binding.to_canonical_json().expect("canonical JSON"), + fixture("remote_initialized_function_binding.canonical.json").trim() + ); +} + #[test] fn refresh_job_result_matches_shared_canonical_golden() { let result = job_result("remote_refresh_job.json"); diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.canonical.json new file mode 100644 index 000000000..7efcecdba --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.canonical.json @@ -0,0 +1 @@ +{"function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"openai_embed","object_id":"fixture","version":"1"},"initialization":{"dimensions":512,"model":"text-embedding-3-small","prompt":null,"retry":{"backoff":[1,2,4],"max_attempts":6},"temperature":0.25},"inputs":[{"kind":"column","parameter":"text","value":{"path":"body"}}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.json new file mode 100644 index 000000000..84d7c48c1 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_application.json @@ -0,0 +1,37 @@ +{ + "function": { + "name": "openai_embed", + "version": "1", + "object_id": "fixture", + "location": "memory:///fixture", + "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6" + }, + "inputs": [ + { + "parameter": "text", + "kind": "column", + "value": { + "path": "body" + } + } + ], + "output": { + "kind": "scalar", + "arrow_type": "list", + "nullable": false + }, + "initialization": { + "model": "text-embedding-3-small", + "dimensions": 512, + "temperature": 0.25, + "retry": { + "max_attempts": 6, + "backoff": [ + 1, + 2, + 4 + ] + }, + "prompt": null + } +} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.canonical.json new file mode 100644 index 000000000..3ffb509d0 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.canonical.json @@ -0,0 +1 @@ +{"binding_id":"fb_01K3EMBED","function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"openai_embed","object_id":"fixture","version":"1"},"initialization":"/////7AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAIAAABUAAAABAAAAMT///8AAAACEAAAACQAAAAEAAAAAAAAAAoAAABkaW1lbnNpb25zAAAIAAwACAAHAAgAAAAAAAABIAAAABAAFAAIAAAABwAMAAAAEAAQAAAAAAAABRAAAAAcAAAABAAAAAAAAAAFAAAAbW9kZWwAAAAEAAQABAAAAP/////IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAAAoAAAAAAAAAAAACgAYAAwABAAIAAoAAABsAAAAEAAAAAEAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAFgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAEAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAFgAAAHRleHQtZW1iZWRkaW5nLTMtc21hbGwAAAACAAAAAAAA/////wAAAAA=","input_schema":{"fields":[{"name":"text","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"text"}],"output_schema":{"fields":[{"name":"embedding","nullable":true,"type":{"fields":[{"name":"item","nullable":false,"type":{"type":"float32"}}],"type":"list"}}]},"outputs":[{"arrow_type":"list","nullable":false,"output_field_id":21,"output_name":"embedding","output_ordinal":0,"result_field":"result"}]} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json new file mode 100644 index 000000000..a81f9321e --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json @@ -0,0 +1,61 @@ +{ + "binding_id": "fb_01K3EMBED", + "function": { + "name": "openai_embed", + "version": "1", + "object_id": "fixture", + "location": "memory:///fixture", + "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6" + }, + "inputs": [ + { + "parameter": "text", + "field_id": 12, + "field_path": "body", + "arrow_type": "utf8", + "nullable": true + } + ], + "outputs": [ + { + "result_field": "result", + "output_name": "embedding", + "output_field_id": 21, + "output_ordinal": 0, + "arrow_type": "list", + "nullable": false + } + ], + "input_schema": { + "fields": [ + { + "name": "text", + "nullable": true, + "type": { + "type": "utf8" + } + } + ] + }, + "output_schema": { + "fields": [ + { + "name": "embedding", + "nullable": true, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + }, + "initialization": "/////7AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAIAAABUAAAABAAAAMT///8AAAACEAAAACQAAAAEAAAAAAAAAAoAAABkaW1lbnNpb25zAAAIAAwACAAHAAgAAAAAAAABIAAAABAAFAAIAAAABwAMAAAAEAAQAAAAAAAABRAAAAAcAAAABAAAAAAAAAAFAAAAbW9kZWwAAAAEAAQABAAAAP/////IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAAAoAAAAAAAAAAAACgAYAAwABAAIAAoAAABsAAAAEAAAAAEAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAFgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAEAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAFgAAAHRleHQtZW1iZWRkaW5nLTMtc21hbGwAAAACAAAAAAAA/////wAAAAA=" +}