mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-18 01:32:27 +00:00
feat: carry Function initialization in the canonical model (#4253)
A Function instance is created once with an initialization row (`create(initialization, context)` in the Function Format), but nothing in the client model could say what that row holds or where its values come from, so every source-built Function ran with an empty row and users pushed configuration through `env=` strings instead (ENT-2516). This adds initialization to the canonical wire model without changing a Function's identity: - `FunctionSignature.initialization` lists the fields of the row, in order. It is omitted when empty, so existing signatures and FunctionVersion hashes are unchanged. - `FunctionApplication.initialization` carries a binding's values as JSON. The service validates them against the Function's fields and re-encodes them as Arrow, so this JSON is transport only and never hashed; that is why floats are accepted here while column-input literals keep the Slice 1 domain. - `FunctionBinding.initialization` is the validated one-row Arrow IPC stream (base64) that every instance of the binding is created with. Values belong to the binding, so one Function version can back several columns with different values. - The binding-shape check that guards schema mutations accepts the new field; without it a table holding an initialized binding refused further declarations. Rust and Python share new canonical goldens for an initialized application and binding. The authoring side (class `@udf`, initialization arguments, shipped helper code) is the stacked PR on top of this one.
This commit is contained in:
Xuanwo
@@ -187,6 +187,11 @@ pub struct FunctionOutput {
|
||||
pub struct FunctionSignature {
|
||||
pub inputs: Vec<FunctionParameter>,
|
||||
pub output: FunctionOutput,
|
||||
/// Ordered fields of the single initialization row a Function instance is
|
||||
/// created with. Each binding supplies its own values; they are not part
|
||||
/// of the Function version. Empty when the Function takes none.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub initialization: Vec<FunctionParameter>,
|
||||
}
|
||||
|
||||
/// One Python environment source.
|
||||
@@ -579,6 +584,13 @@ pub struct FunctionApplication {
|
||||
output: FunctionOutput,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
columns: BTreeMap<String, String>,
|
||||
/// Initialization values for this binding, by initialization field name.
|
||||
/// The service validates them against the Function's initialization
|
||||
/// schema and persists the resulting Arrow row in the binding; this JSON
|
||||
/// is transport only and never hashed into an identity, so floating-point
|
||||
/// values are accepted here.
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
initialization: BTreeMap<String, Value>,
|
||||
#[serde(default, flatten, skip_serializing)]
|
||||
unknown_fields: BTreeMap<String, Value>,
|
||||
#[serde(default, skip)]
|
||||
@@ -601,6 +613,11 @@ impl FunctionApplication {
|
||||
pub fn columns(&self) -> &BTreeMap<String, String> {
|
||||
&self.columns
|
||||
}
|
||||
|
||||
/// Initialization values supplied by this application, by field name.
|
||||
pub fn initialization(&self) -> &BTreeMap<String, Value> {
|
||||
&self.initialization
|
||||
}
|
||||
/// Whether a newer writer attached application fields this client cannot
|
||||
/// validate. Such applications remain readable but must not be declared.
|
||||
pub fn has_unknown_fields(&self) -> bool {
|
||||
@@ -677,6 +694,11 @@ pub struct FunctionBinding {
|
||||
/// Exact physical Arrow schema of the binding's table outputs.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
output_schema: Option<Value>,
|
||||
/// Standard base64 of the one-row Arrow IPC stream every instance of this
|
||||
/// binding is created with, already validated against the Function's
|
||||
/// initialization schema. Absent when the Function takes none.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
initialization: Option<String>,
|
||||
}
|
||||
|
||||
impl FunctionBinding {
|
||||
@@ -707,6 +729,11 @@ impl FunctionBinding {
|
||||
pub fn output_schema(&self) -> Option<&Value> {
|
||||
self.output_schema.as_ref()
|
||||
}
|
||||
|
||||
/// Base64 one-row Arrow IPC initialization stream, if the Function takes one.
|
||||
pub fn initialization(&self) -> Option<&str> {
|
||||
self.initialization.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
impl_json!(FunctionBinding);
|
||||
@@ -964,6 +991,66 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// A Function without initialization keeps its pre-initialization wire
|
||||
/// form, so existing signatures and applications encode byte-identically.
|
||||
#[test]
|
||||
fn empty_initialization_is_omitted_from_the_wire() {
|
||||
let signature: FunctionSignature = serde_json::from_str(
|
||||
r#"{"inputs":[{"name":"x","arrow_type":"int64","nullable":true}],"output":{"kind":"scalar","arrow_type":"int64","nullable":false}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(signature.initialization.is_empty());
|
||||
assert!(
|
||||
!canonical_json(&signature)
|
||||
.unwrap()
|
||||
.contains("initialization")
|
||||
);
|
||||
|
||||
let signature: FunctionSignature = serde_json::from_str(
|
||||
r#"{"inputs":[],"output":{"kind":"scalar","arrow_type":"int64","nullable":false},"initialization":[{"name":"factor","arrow_type":"float64","nullable":false}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(signature.initialization[0].name, "factor");
|
||||
assert!(
|
||||
canonical_json(&signature)
|
||||
.unwrap()
|
||||
.contains(r#""initialization":[{"#)
|
||||
);
|
||||
}
|
||||
|
||||
/// Initialization values are transport JSON, validated and re-encoded as
|
||||
/// Arrow by the service, so floats are accepted there while column-input
|
||||
/// literals keep the Slice 1 domain.
|
||||
#[test]
|
||||
fn application_initialization_accepts_floats() {
|
||||
let application = FunctionApplication::from_json(
|
||||
r#"{"function":{"name":"scale","object_id":"o","location":"l","version":"1","manifest_digest":"sha256:x"},"inputs":[{"parameter":"x","kind":"column","value":{"path":"x"}}],"output":{"kind":"scalar","arrow_type":"float64","nullable":false},"initialization":{"factor":2.5,"label":"fast"}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!application.has_unknown_fields());
|
||||
assert_eq!(
|
||||
application.initialization()["factor"],
|
||||
serde_json::json!(2.5)
|
||||
);
|
||||
let encoded = application.to_canonical_json().unwrap();
|
||||
assert!(encoded.contains(r#""initialization":{"factor":2.5,"label":"fast"}"#));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn binding_initialization_round_trips() {
|
||||
let binding = FunctionBinding::from_json(
|
||||
r#"{"binding_id":"fb_1","function":{"name":"scale","object_id":"o","location":"l","version":"1","manifest_digest":"sha256:x"},"inputs":[],"outputs":[],"initialization":"QUJD"}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(binding.initialization(), Some("QUJD"));
|
||||
assert!(
|
||||
binding
|
||||
.to_canonical_json()
|
||||
.unwrap()
|
||||
.ends_with(r#""initialization":"QUJD","inputs":[],"outputs":[]}"#)
|
||||
);
|
||||
}
|
||||
|
||||
/// Unknown keys are how a newer server's payload reaches an older client,
|
||||
/// so the check has to be exact about which level it is looking at.
|
||||
#[test]
|
||||
|
||||
@@ -695,6 +695,7 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
|
||||
"assignment",
|
||||
"input_schema",
|
||||
"output_schema",
|
||||
"initialization",
|
||||
],
|
||||
"binding",
|
||||
)?;
|
||||
@@ -3748,6 +3749,17 @@ mod tests {
|
||||
assert!(matches!(err, Error::NotSupported { .. }));
|
||||
}
|
||||
|
||||
/// A binding's initialization row is part of the known contract, so a
|
||||
/// table that holds one can still take further declarations.
|
||||
#[test]
|
||||
fn test_initialized_bindings_are_a_known_shape() {
|
||||
let raw_binding: Value = serde_json::from_str(include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_initialized_function_binding.json"
|
||||
))
|
||||
.unwrap();
|
||||
ensure_known_binding_shape(&raw_binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_named_struct_can_be_kept_as_one_nullable_physical_column() {
|
||||
let application = named_struct_application("{}");
|
||||
|
||||
@@ -115,6 +115,29 @@ fn application_and_binding_match_shared_remote_goldens() {
|
||||
);
|
||||
}
|
||||
|
||||
/// Initialization values travel as JSON on the application, including
|
||||
/// floats, and as the validated Arrow row on the binding.
|
||||
#[test]
|
||||
fn initialized_application_and_binding_match_shared_remote_goldens() {
|
||||
let application =
|
||||
FunctionApplication::from_json(&fixture("remote_initialized_function_application.json"))
|
||||
.expect("application fixture");
|
||||
assert!(!application.has_unknown_fields());
|
||||
assert_eq!(application.initialization()["temperature"], 0.25);
|
||||
assert_eq!(
|
||||
application.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_initialized_function_application.canonical.json").trim()
|
||||
);
|
||||
|
||||
let binding = FunctionBinding::from_json(&fixture("remote_initialized_function_binding.json"))
|
||||
.expect("binding fixture");
|
||||
assert!(binding.initialization().is_some());
|
||||
assert_eq!(
|
||||
binding.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_initialized_function_binding.canonical.json").trim()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_job_result_matches_shared_canonical_golden() {
|
||||
let result = job_result("remote_refresh_job.json");
|
||||
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"openai_embed","object_id":"fixture","version":"1"},"initialization":{"dimensions":512,"model":"text-embedding-3-small","prompt":null,"retry":{"backoff":[1,2,4],"max_attempts":6},"temperature":0.25},"inputs":[{"kind":"column","parameter":"text","value":{"path":"body"}}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}}
|
||||
Vendored
+37
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"function": {
|
||||
"name": "openai_embed",
|
||||
"version": "1",
|
||||
"object_id": "fixture",
|
||||
"location": "memory:///fixture",
|
||||
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"parameter": "text",
|
||||
"kind": "column",
|
||||
"value": {
|
||||
"path": "body"
|
||||
}
|
||||
}
|
||||
],
|
||||
"output": {
|
||||
"kind": "scalar",
|
||||
"arrow_type": "list<float32>",
|
||||
"nullable": false
|
||||
},
|
||||
"initialization": {
|
||||
"model": "text-embedding-3-small",
|
||||
"dimensions": 512,
|
||||
"temperature": 0.25,
|
||||
"retry": {
|
||||
"max_attempts": 6,
|
||||
"backoff": [
|
||||
1,
|
||||
2,
|
||||
4
|
||||
]
|
||||
},
|
||||
"prompt": null
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"binding_id":"fb_01K3EMBED","function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"openai_embed","object_id":"fixture","version":"1"},"initialization":"/////7AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAIAAABUAAAABAAAAMT///8AAAACEAAAACQAAAAEAAAAAAAAAAoAAABkaW1lbnNpb25zAAAIAAwACAAHAAgAAAAAAAABIAAAABAAFAAIAAAABwAMAAAAEAAQAAAAAAAABRAAAAAcAAAABAAAAAAAAAAFAAAAbW9kZWwAAAAEAAQABAAAAP/////IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAAAoAAAAAAAAAAAACgAYAAwABAAIAAoAAABsAAAAEAAAAAEAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAFgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAEAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAFgAAAHRleHQtZW1iZWRkaW5nLTMtc21hbGwAAAACAAAAAAAA/////wAAAAA=","input_schema":{"fields":[{"name":"text","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"text"}],"output_schema":{"fields":[{"name":"embedding","nullable":true,"type":{"fields":[{"name":"item","nullable":false,"type":{"type":"float32"}}],"type":"list"}}]},"outputs":[{"arrow_type":"list<float32>","nullable":false,"output_field_id":21,"output_name":"embedding","output_ordinal":0,"result_field":"result"}]}
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
{
|
||||
"binding_id": "fb_01K3EMBED",
|
||||
"function": {
|
||||
"name": "openai_embed",
|
||||
"version": "1",
|
||||
"object_id": "fixture",
|
||||
"location": "memory:///fixture",
|
||||
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"
|
||||
},
|
||||
"inputs": [
|
||||
{
|
||||
"parameter": "text",
|
||||
"field_id": 12,
|
||||
"field_path": "body",
|
||||
"arrow_type": "utf8",
|
||||
"nullable": true
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"result_field": "result",
|
||||
"output_name": "embedding",
|
||||
"output_field_id": 21,
|
||||
"output_ordinal": 0,
|
||||
"arrow_type": "list<float32>",
|
||||
"nullable": false
|
||||
}
|
||||
],
|
||||
"input_schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "text",
|
||||
"nullable": true,
|
||||
"type": {
|
||||
"type": "utf8"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"output_schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "embedding",
|
||||
"nullable": true,
|
||||
"type": {
|
||||
"type": "list",
|
||||
"fields": [
|
||||
{
|
||||
"name": "item",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": "float32"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"initialization": "/////7AAAAAQAAAAAAAKAAwABgAFAAgACgAAAAABBAAMAAAACAAIAAAABAAIAAAABAAAAAIAAABUAAAABAAAAMT///8AAAACEAAAACQAAAAEAAAAAAAAAAoAAABkaW1lbnNpb25zAAAIAAwACAAHAAgAAAAAAAABIAAAABAAFAAIAAAABwAMAAAAEAAQAAAAAAAABRAAAAAcAAAABAAAAAAAAAAFAAAAbW9kZWwAAAAEAAQABAAAAP/////IAAAAFAAAAAAAAAAMABYABgAFAAgADAAMAAAAAAMEABgAAAAoAAAAAAAAAAAACgAYAAwABAAIAAoAAABsAAAAEAAAAAEAAAAAAAAAAAAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAgAAAAAAAAAFgAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAEAAAAAAAAAAAAAAACAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAFgAAAHRleHQtZW1iZWRkaW5nLTMtc21hbGwAAAACAAAAAAAA/////wAAAAA="
|
||||
}
|
||||
Reference in New Issue
Block a user