rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 22:26:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: represent registered Functions by OCI image identity

Browse Source
This commit is contained in:
Xuanwo
2026-09-15 02:52:26 +08:00
parent c44b192334
commit ac19aefe3c
10 changed files with 130 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/python/python.md
+4
View File
@@ -133,10 +133,14 @@ listing a storage directory.
::: lancedb.functions.PythonAdapterSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersion
::: lancedb.functions.PythonRuntimeSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersionRef
::: lancedb.functions.ApplicationInput
python/python/lancedb/functions.py
+11 -9
View File
@@ -295,20 +295,21 @@ class PythonRuntimeSpec(_RemoteValue):
return self
class FunctionVersion(_RemoteValue):
"""An exact immutable Function version returned by Enterprise.
class FunctionImage(_RemoteValue):
"""A complete OCI Function image identified by its exact manifest digest."""
The GPU execution requirement is part of this identity. CPU and memory sizing,
priority, concurrency, and retry policy belong to the execution platform.
"""
manifest_digest: str
descriptor: Mapping[str, Any]
source: bool
class FunctionVersion(_RemoteValue):
"""An immutable catalog name bound to a concrete OCI Function image."""
name: str
version: str
artifact: FunctionArtifact
image: FunctionImage
signature: FunctionSignature
runtime: PythonRuntimeSpec
runtime_digest: str
environment_digest: str
created_at: str
def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -1402,6 +1403,7 @@ __all__ = [
"FunctionRegistrationRequest",
"FunctionResultField",
"FunctionSignature",
"FunctionImage",
"FunctionVersion",
"FunctionVersionRef",
"InputBinding",
python/python/tests/test_first_class_function_slice1.py
+8 -7
View File
@@ -93,12 +93,13 @@ def test_function_version_identity_is_immutable_and_exact():
value = job_result("remote_function_job.json")
version = FunctionVersion.from_json(json.dumps(value))
assert version.name == "embed"
assert version.version == "fv_01K3EXACT"
assert version.version == version.image.manifest_digest
assert version.version.startswith("sha256:")
with pytest.raises((TypeError, ValueError)):
version.version = "fv_changed"
with pytest.raises(TypeError, match="immutable"):
version.runtime.env["TOKENIZERS_PARALLELISM"] = "true"
version.image.descriptor["format_version"] = "changed"
changed = dict(value)
changed["version"] = "fv_changed"
@@ -182,14 +183,14 @@ def test_function_version_keeps_named_struct_outputs_in_one_application():
def test_unknown_fields_and_discriminators_are_forward_decodable():
value = job_result("remote_function_job.json")
value["future_version_metadata"] = {"retention_class": "catalog"}
value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"}
value["image"]["descriptor"]["future_interface"] = {"kind": "wasm"}
value["signature"]["output"]["kind"] = "future_output_shape"
version = FunctionVersion.from_json(json.dumps(value))
assert version.runtime.kind == "wasm"
assert version.runtime.python_version is None
assert version.runtime.environment is None
assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"}
assert version.image.descriptor["future_interface"] == {"kind": "wasm"}
assert json.loads(version.to_canonical_json())["image"]["descriptor"][
"future_interface"
] == {"kind": "wasm"}
assert version.signature.output.kind == "future_output_shape"
python/python/tests/test_first_class_function_slice2.py
+22 -19
View File
@@ -45,6 +45,11 @@ FIXTURES = (
)
IMAGE_VERSION = json.loads(
(FIXTURES / "remote_function_version.canonical.json").read_text()
)["version"]
@udf(
pip=["numpy>=2"],
env={"MODE": "test"},
@@ -1199,11 +1204,11 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
with pytest.raises(NotImplementedError, match=message):
db.create_function_async(normalize_score)
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version="fv_exact")
db.get_function("normalize_score", version=IMAGE_VERSION)
with pytest.raises(NotImplementedError, match=message):
db.list_functions()
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version="fv_exact")
db.drop_function("normalize_score", version=IMAGE_VERSION)
@contextlib.contextmanager
@@ -1230,15 +1235,13 @@ def _mock_remote_function_catalog():
if self.path == "/v1/function/normalize_score/create":
state["version"] = {
"name": "normalize_score",
"version": "fv_exact",
"artifact": {
key: body["artifact"][key]
for key in ("kind", "digest", "entrypoint")
},
"version": IMAGE_VERSION,
"image": json.loads(
(
FIXTURES / "remote_function_version.canonical.json"
).read_text()
)["image"],
"signature": body["signature"],
"runtime": body["runtime"],
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"created_at": "2026-08-21T00:00:00Z",
}
response = {"job_id": "job-register"}
@@ -1252,10 +1255,10 @@ def _mock_remote_function_catalog():
"result": state["version"],
}
elif self.path == "/v1/function/normalize_score/describe":
assert body == {"version": "fv_exact"}
assert body == {"version": IMAGE_VERSION}
response = state["version"]
elif self.path == "/v1/function/normalize_score/drop":
assert body == {"version": "fv_exact"}
assert body == {"version": IMAGE_VERSION}
response = {"dropped": True}
else:
status = 404
@@ -1278,7 +1281,7 @@ def _mock_remote_function_catalog():
"functions": [
{
"name": "normalize_score",
"version": "fv_exact",
"version": IMAGE_VERSION,
"definition": state["version"],
}
],
@@ -1314,7 +1317,7 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
assert created == reopened
assert reopened.name == "normalize_score"
assert reopened.version == "fv_exact"
assert reopened.version == IMAGE_VERSION
create_request = state["requests"][0][1]
expected_request = json.loads(
normalize_score.registration_request.to_canonical_json()
@@ -1334,7 +1337,7 @@ def test_blocking_remote_registration_returns_function_version():
created = db.create_function(normalize_score)
assert created.name == "normalize_score"
assert created.version == "fv_exact"
assert created.version == IMAGE_VERSION
assert [path for path, _ in state["requests"]] == [
"/v1/function/normalize_score/create",
"/v1/jobs/describe",
@@ -1392,12 +1395,12 @@ def test_remote_drop_function_sends_exact_version():
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.drop_function("normalize_score", version="fv_exact") is True
assert db.drop_function("normalize_score", version=IMAGE_VERSION) is True
assert state["requests"] == [
(
"/v1/function/normalize_score/drop",
{"version": "fv_exact"},
{"version": IMAGE_VERSION},
)
]
@@ -1411,11 +1414,11 @@ async def test_async_remote_drop_function_sends_exact_version():
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert await db.drop_function("normalize_score", version="fv_exact") is True
assert await db.drop_function("normalize_score", version=IMAGE_VERSION) is True
assert state["requests"] == [
(
"/v1/function/normalize_score/drop",
{"version": "fv_exact"},
{"version": IMAGE_VERSION},
)
]
rust/lancedb/src/function.rs
+15 -31
View File
@@ -396,19 +396,21 @@ impl Serialize for PythonRuntimeSpec {
}
}
/// Immutable Function version returned by the Enterprise catalog.
///
/// The GPU execution requirement is part of this identity. CPU and memory sizing,
/// priority, concurrency, and retry policy belong to the execution platform.
/// A complete OCI Function image. Its digest is independent of catalog names.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionImage {
pub manifest_digest: String,
pub descriptor: Value,
pub source: bool,
}
/// An immutable catalog name bound to a concrete OCI manifest.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersion {
name: String,
version: String,
artifact: FunctionArtifact,
image: FunctionImage,
signature: FunctionSignature,
runtime: PythonRuntimeSpec,
runtime_digest: String,
environment_digest: String,
created_at: String,
}
@@ -416,31 +418,15 @@ impl FunctionVersion {
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> &str {
&self.version
}
pub fn artifact(&self) -> &FunctionArtifact {
&self.artifact
pub fn image(&self) -> &FunctionImage {
&self.image
}
pub fn signature(&self) -> &FunctionSignature {
&self.signature
}
pub fn runtime(&self) -> &PythonRuntimeSpec {
&self.runtime
}
pub fn runtime_digest(&self) -> &str {
&self.runtime_digest
}
pub fn environment_digest(&self) -> &str {
&self.environment_digest
}
pub fn created_at(&self) -> &str {
&self.created_at
}
@@ -697,11 +683,9 @@ mod conda_environment_tests {
)
.unwrap();
assert_eq!(conda.channels, ["conda-forge"]);
assert!(
serde_json::to_string(&conda)
.unwrap()
.contains(r#""channels":["conda-forge"]"#)
);
assert!(serde_json::to_string(&conda)
.unwrap()
.contains(r#""channels":["conda-forge"]"#));
let pip: PythonEnvironmentSpec =
serde_json::from_str(r#"{"kind":"pip","packages":["numpy"]}"#).unwrap();
rust/lancedb/tests/first_class_function_slice1.rs
+12 -15
View File
@@ -26,8 +26,11 @@ fn function_version_job_result_matches_shared_canonical_golden() {
let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
assert_eq!(version.name(), "embed");
assert_eq!(version.version(), "fv_01K3EXACT");
assert_eq!(version.runtime_digest(), "sha256:runtime");
assert_eq!(
version.version(),
"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"
);
assert_eq!(version.image().manifest_digest, version.version());
assert_eq!(
version.to_canonical_json().expect("canonical JSON"),
fixture("remote_function_version.canonical.json").trim()
@@ -113,22 +116,18 @@ fn refresh_job_result_matches_shared_canonical_golden() {
fn unknown_fields_and_discriminators_are_forward_decodable() {
let mut result = job_result("remote_function_job.json");
result["future_version_metadata"] = serde_json::json!({"retention_class": "catalog"});
result["runtime"] = serde_json::json!({
"kind": "wasm",
"module_digest": "sha256:wasm"
});
result["image"]["descriptor"]["future_interface"] = serde_json::json!({"version": 2});
result["signature"]["output"]["kind"] = Value::String("future_output_shape".to_string());
let version = FunctionVersion::from_json(&result.to_string()).expect("future remote value");
assert_eq!(version.runtime().kind(), "wasm");
assert_eq!(version.runtime().python_version(), None);
assert_eq!(version.image().descriptor["future_interface"]["version"], 2);
assert_eq!(version.signature().output.kind, "future_output_shape");
assert_eq!(
serde_json::from_str::<Value>(
&version.to_canonical_json().expect("canonical future value")
)
.expect("canonical JSON")["runtime"],
serde_json::json!({"kind": "wasm"})
.expect("canonical JSON")["image"]["descriptor"]["future_interface"],
serde_json::json!({"version": 2})
);
}
@@ -136,9 +135,7 @@ fn unknown_fields_and_discriminators_are_forward_decodable() {
fn floating_point_application_literals_are_rejected_consistently() {
let error = FunctionApplication::from_json(&fixture("remote_function_application_float.json"))
.unwrap_err();
assert!(
error
.to_string()
.contains("floating-point Function literals")
);
assert!(error
.to_string()
.contains("floating-point Function literals"));
}
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json
Vendored
+1 -1
View File
@@ -9,7 +9,7 @@
"application": {
"function": {
"name": "embed",
"version": "fv_01K3EXACT"
"version": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"
},
"inputs": [
{
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json
Vendored
+55 -19
View File
@@ -3,28 +3,64 @@
"job_type": "create_function",
"job_state": "DONE",
"creation_ms": 1787270400000,
"spec": {"name": "embed"},
"spec": {
"name": "embed"
},
"result": {
"name": "embed",
"version": "fv_01K3EXACT",
"artifact": {
"kind": "python_callable",
"digest": "sha256:code",
"entrypoint": "embed"
},
"version": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6",
"signature": {
"inputs": [{"name": "text", "arrow_type": "utf8", "nullable": true}],
"output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false}
"inputs": [
{
"name": "text",
"arrow_type": "utf8",
"nullable": true
}
],
"output": {
"kind": "scalar",
"arrow_type": "list<float32>",
"nullable": false
}
},
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {"kind": "pip", "packages": ["sentence-transformers>=3"]},
"env": {"TOKENIZERS_PARALLELISM": "false"}
},
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"created_at": "2026-08-21T00:00:00Z"
"created_at": "2026-08-21T00:00:00Z",
"image": {
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6",
"descriptor": {
"format_version": 1,
"python": {
"implementation": "cpython",
"version": "3.12.14",
"abi_tag": "cp312",
"executable": "/usr/local/bin/python3",
"import_paths": [
"/opt/function/code"
]
},
"python_api": 1,
"entrypoint": "app.function:create",
"interface": {
"type": "lance.scalar",
"version": 1
},
"schemas": {
"input": "/opt/function/schemas/input.arrow",
"output": "/opt/function/schemas/output.arrow",
"initialization": "/opt/function/schemas/initialization.arrow"
},
"requires": {
"kernel_min": "4.18.0",
"capabilities": []
},
"behavior": {
"result_stability": "input_and_context",
"side_effects": "non_idempotent"
}
},
"source": false
}
},
"future_job": {"trace_id": "trace-1"}
"future_job": {
"trace_id": "trace-1"
}
}
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json
Vendored
+1 -1
View File
@@ -1 +1 @@
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
{"created_at":"2026-08-21T00:00:00Z","image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"name":"embed","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json
Vendored
+1 -1
View File
@@ -4,7 +4,7 @@
],
"function": {
"application": {
"function": {"name": "embed", "version": "fv_01K3EXACT"},
"function": {"name": "embed", "version": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [
{"parameter": "text", "kind": "column", "value": {"path": "description"}}
],