feat: represent registered Functions by OCI image identity (#4176)

Function versions identify independent Function objects and their
numeric revisions. Rust and Python expose the object ID, location,
canonical decimal version, metadata, and availability separately from
the OCI image digest. Computed-column applications carry the complete
object reference, so existing bindings retain their identity after a
name is removed and reused.

Source authoring keeps the existing
create_function/create_function_async, job wait, and column-binding
APIs. The server coordinates baking followed by registration; users do
not have to manage manifest digests to create a Function. The previous
stored Function representation is intentionally unsupported. Existing
contract tests and shared wire fixtures are migrated to the new model.

This SDK change accompanies the final integration layer
https://github.com/lancedb/sophon/pull/7887 in the Sophon stack:
https://github.com/lancedb/sophon/pull/7885https://github.com/lancedb/sophon/pull/7886https://github.com/lancedb/sophon/pull/7887. A metadata-only revision
can retain the same executable image; Function version numbers must not
be used as image cache keys.

---------

Co-authored-by: lancedb automation <robot@lancedb.com>
Co-authored-by: Yang Cen <bubble-cal@outlook.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Xuanwo
2026-09-16 12:38:59 +08:00
committed by GitHub
co-authored by lancedb automation Yang Cen Claude Fable 5.1
parent f7579d2aae
commit 3b37ea2c7a
27 changed files with 314 additions and 165 deletions
+2
View File
@@ -41,6 +41,8 @@ arrow-select = "58.0.0"
arrow-cast = "58.0.0"
arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] }
async-trait = "0"
# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin.
aws-smithy-types = ">=1.3.6, <1.7"
bytes = "1"
datafusion = { version = "54.0.0", default-features = false }
datafusion-catalog = "54.0.0"
+2
View File
@@ -133,6 +133,8 @@ listing a storage directory.
::: lancedb.functions.PythonAdapterSpec
::: lancedb.functions.FunctionImage
::: lancedb.functions.FunctionVersion
::: lancedb.functions.PythonRuntimeSpec
+15 -11
View File
@@ -743,19 +743,20 @@ class DBConnection(EnforceOverrides):
raise NotImplementedError("serialize is not supported for this connection type")
def create_function(self, definition: UdfDefinition) -> FunctionVersion:
"""Register a scalar Python UDF and wait for its immutable version.
"""Build and register a scalar Python UDF, then return its version.
The server builds the OCI image and registers the completed artifact.
This is the blocking counterpart of :meth:`create_function_async`.
Local connections raise ``NotImplementedError``.
"""
return self.create_function_async(definition).wait()
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
"""Register a scalar Python UDF through the remote Function catalog.
"""Submit a scalar Python UDF for building and registration.
Submission returns a typed job. The immutable Function version becomes
available only when :meth:`Job.wait` succeeds. Local connections raise
``NotImplementedError``.
The server-side job builds the OCI image, then registers the completed
artifact. Waiting on the job returns the immutable Function version.
Local connections raise ``NotImplementedError``.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
@@ -786,10 +787,12 @@ class DBConnection(EnforceOverrides):
)
def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog.
"""Remove the current Function name binding from the remote catalog.
Returns True when the version changed to Dropped and False for an
idempotent replay. Local connections raise NotImplementedError.
The requested version must exist in the currently named object.
Object history and existing computed-column references are retained.
Returns True when the name was removed and False when it was absent.
Local connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
@@ -2421,9 +2424,10 @@ class AsyncConnection(object):
async def create_function_async(
self, definition: UdfDefinition
) -> AsyncJob[FunctionVersion]:
"""Register a scalar Python UDF through the remote Function catalog.
"""Submit a scalar Python UDF for building and registration.
The returned typed job resolves to the immutable Function version.
The server-side job builds the OCI image, then registers the completed
artifact. Waiting on the job returns the immutable Function version.
Local connections raise ``NotImplementedError``.
"""
if not isinstance(definition, UdfDefinition):
@@ -2449,7 +2453,7 @@ class AsyncConnection(object):
]
async def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog."""
"""Remove the current name binding, retaining the object and its history."""
return await self._inner.drop_function(name, version)
async def list_jobs(self) -> List[JobInfo]:
+41 -12
View File
@@ -41,6 +41,7 @@ from typing import (
import pyarrow as pa
from pydantic import (
AfterValidator,
BaseModel,
ConfigDict,
Field,
@@ -295,21 +296,39 @@ class PythonRuntimeSpec(_RemoteValue):
return self
class FunctionVersion(_RemoteValue):
"""An exact immutable Function version returned by Enterprise.
class FunctionImage(_RemoteValue):
"""A complete OCI Function image identified by its exact manifest digest."""
The GPU execution requirement is part of this identity. CPU and memory sizing,
priority, concurrency, and retry policy belong to the execution platform.
"""
manifest_digest: str
descriptor: Mapping[str, Any]
source: bool
def _validate_object_version(value: str) -> str:
if int(value) > 2**64 - 1:
raise ValueError("Function version exceeds uint64")
return value
_ObjectVersion = Annotated[
str,
Field(strict=True, pattern=r"^[1-9][0-9]*$"),
AfterValidator(_validate_object_version),
]
class FunctionVersion(_RemoteValue):
"""A pinned object revision, independent of its executable image digest."""
name: str
version: str
artifact: FunctionArtifact
object_id: str
location: str
version: _ObjectVersion
image: FunctionImage
signature: FunctionSignature
runtime: PythonRuntimeSpec
runtime_digest: str
environment_digest: str
created_at: str
metadata: Mapping[str, str]
disabled: bool
def __call__(self, **inputs: Any) -> FunctionApplication:
"""Bind this exact version to named table columns.
@@ -363,7 +382,13 @@ class FunctionVersion(_RemoteValue):
)
)
return FunctionApplication(
function=FunctionVersionRef(name=self.name, version=self.version),
function=FunctionVersionRef(
name=self.name,
object_id=self.object_id,
location=self.location,
version=self.version,
manifest_digest=self.image.manifest_digest,
),
inputs=tuple(bindings),
output=self.signature.output,
)
@@ -380,7 +405,10 @@ class FunctionRegistrationRequest(_RemoteValue):
class FunctionVersionRef(_OpenRemoteValue):
name: str
version: str
object_id: str
location: str
version: _ObjectVersion
manifest_digest: str
class ApplicationInput(_OpenRemoteValue):
@@ -1402,6 +1430,7 @@ __all__ = [
"FunctionRegistrationRequest",
"FunctionResultField",
"FunctionSignature",
"FunctionImage",
"FunctionVersion",
"FunctionVersionRef",
"InputBinding",
@@ -93,16 +93,22 @@ def test_function_version_identity_is_immutable_and_exact():
value = job_result("remote_function_job.json")
version = FunctionVersion.from_json(json.dumps(value))
assert version.name == "embed"
assert version.version == "fv_01K3EXACT"
assert version.version == "1"
assert version.image.manifest_digest.startswith("sha256:")
assert version.version != version.image.manifest_digest
with pytest.raises((TypeError, ValueError)):
version.version = "fv_changed"
version.version = "1"
with pytest.raises(TypeError, match="immutable"):
version.runtime.env["TOKENIZERS_PARALLELISM"] = "true"
version.image.descriptor["format_version"] = "changed"
changed = dict(value)
changed["version"] = "fv_changed"
changed["version"] = "2"
assert FunctionVersion(**changed) != version
assert FunctionVersion(**changed).image == version.image
for invalid in [version.image.manifest_digest, "0", "01", "-1", str(2**64)]:
with pytest.raises(ValueError):
FunctionVersion(**{**value, "version": invalid})
def test_function_version_binds_named_columns_as_one_immutable_application():
@@ -137,7 +143,7 @@ def test_function_version_binding_validates_names_and_direct_columns():
def test_function_version_keeps_named_struct_outputs_in_one_application():
value = job_result("remote_function_job.json")
value["name"] = "text_features"
value["version"] = "fv_multi_output"
value["version"] = "1"
value["signature"] = {
"inputs": [
{"name": "title", "arrow_type": "utf8", "nullable": True},
@@ -182,14 +188,14 @@ def test_function_version_keeps_named_struct_outputs_in_one_application():
def test_unknown_fields_and_discriminators_are_forward_decodable():
value = job_result("remote_function_job.json")
value["future_version_metadata"] = {"retention_class": "catalog"}
value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"}
value["image"]["descriptor"]["future_interface"] = {"kind": "wasm"}
value["signature"]["output"]["kind"] = "future_output_shape"
version = FunctionVersion.from_json(json.dumps(value))
assert version.runtime.kind == "wasm"
assert version.runtime.python_version is None
assert version.runtime.environment is None
assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"}
assert version.image.descriptor["future_interface"] == {"kind": "wasm"}
assert json.loads(version.to_canonical_json())["image"]["descriptor"][
"future_interface"
] == {"kind": "wasm"}
assert version.signature.output.kind == "future_output_shape"
@@ -222,7 +228,7 @@ def test_function_application_uses_rename_columns_only():
def test_binding_and_refresh_result_keep_stable_remote_fields():
binding = FunctionBinding.from_json(fixture("remote_function_binding.json"))
assert binding.function.version == "fv_01K3TEXT"
assert binding.function.version == "1"
assert [output.output_ordinal for output in binding.outputs] == [0, 1]
assert binding.input_schema is not None
assert binding.output_schema is not None
@@ -339,7 +345,16 @@ def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
scalar = FunctionApplication.from_json(
json.dumps(
{
"function": {"name": "embed", "version": "fv_exact"},
"function": {
"name": "embed",
"version": "1",
"object_id": "fixture",
"location": "memory:///fixture",
"manifest_digest": (
"sha256:"
"7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"
),
},
"inputs": [],
"output": {
"kind": "scalar",
@@ -45,6 +45,11 @@ FIXTURES = (
)
FUNCTION_VERSION = json.loads(
(FIXTURES / "remote_function_version.canonical.json").read_text()
)["version"]
@udf(
pip=["numpy>=2"],
env={"MODE": "test"},
@@ -1199,11 +1204,11 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
with pytest.raises(NotImplementedError, match=message):
db.create_function_async(normalize_score)
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version="fv_exact")
db.get_function("normalize_score", version=FUNCTION_VERSION)
with pytest.raises(NotImplementedError, match=message):
db.list_functions()
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version="fv_exact")
db.drop_function("normalize_score", version=FUNCTION_VERSION)
@contextlib.contextmanager
@@ -1230,15 +1235,17 @@ def _mock_remote_function_catalog():
if self.path == "/v1/function/normalize_score/create":
state["version"] = {
"name": "normalize_score",
"version": "fv_exact",
"artifact": {
key: body["artifact"][key]
for key in ("kind", "digest", "entrypoint")
},
"version": FUNCTION_VERSION,
"object_id": "fixture",
"location": "memory:///fixture",
"metadata": {},
"disabled": False,
"image": json.loads(
(
FIXTURES / "remote_function_version.canonical.json"
).read_text()
)["image"],
"signature": body["signature"],
"runtime": body["runtime"],
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"created_at": "2026-08-21T00:00:00Z",
}
response = {"job_id": "job-register"}
@@ -1252,10 +1259,10 @@ def _mock_remote_function_catalog():
"result": state["version"],
}
elif self.path == "/v1/function/normalize_score/describe":
assert body == {"version": "fv_exact"}
assert body == {"version": FUNCTION_VERSION}
response = state["version"]
elif self.path == "/v1/function/normalize_score/drop":
assert body == {"version": "fv_exact"}
assert body == {"version": FUNCTION_VERSION}
response = {"dropped": True}
else:
status = 404
@@ -1278,7 +1285,7 @@ def _mock_remote_function_catalog():
"functions": [
{
"name": "normalize_score",
"version": "fv_exact",
"version": FUNCTION_VERSION,
"definition": state["version"],
}
],
@@ -1314,7 +1321,7 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
assert created == reopened
assert reopened.name == "normalize_score"
assert reopened.version == "fv_exact"
assert reopened.version == FUNCTION_VERSION
create_request = state["requests"][0][1]
expected_request = json.loads(
normalize_score.registration_request.to_canonical_json()
@@ -1334,7 +1341,7 @@ def test_blocking_remote_registration_returns_function_version():
created = db.create_function(normalize_score)
assert created.name == "normalize_score"
assert created.version == "fv_exact"
assert created.version == FUNCTION_VERSION
assert [path for path, _ in state["requests"]] == [
"/v1/function/normalize_score/create",
"/v1/jobs/describe",
@@ -1392,12 +1399,12 @@ def test_remote_drop_function_sends_exact_version():
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.drop_function("normalize_score", version="fv_exact") is True
assert db.drop_function("normalize_score", version=FUNCTION_VERSION) is True
assert state["requests"] == [
(
"/v1/function/normalize_score/drop",
{"version": "fv_exact"},
{"version": FUNCTION_VERSION},
)
]
@@ -1411,11 +1418,13 @@ async def test_async_remote_drop_function_sends_exact_version():
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert await db.drop_function("normalize_score", version="fv_exact") is True
assert (
await db.drop_function("normalize_score", version=FUNCTION_VERSION) is True
)
assert state["requests"] == [
(
"/v1/function/normalize_score/drop",
{"version": "fv_exact"},
{"version": FUNCTION_VERSION},
)
]
+4 -5
View File
@@ -67,6 +67,7 @@ serde_json = { workspace = true }
async-openai = { version = "0.20.0", optional = true }
serde_with = { version = "3.8.1" }
tempfile = { workspace = true }
aws-smithy-types = { workspace = true, optional = true }
aws-sdk-bedrockruntime = { version = "1.27.0", optional = true }
# For remote feature
reqwest = { version = "0.12.0", default-features = false, features = [
@@ -114,10 +115,7 @@ aws-sdk-s3 = { version = "1.55.0" }
aws-sdk-kms = { version = "1.48.0" }
aws-config = { version = "1.5.10" }
aws-smithy-runtime = { version = "1.9.1" }
# Constraint only: types 1.7 breaks aws-smithy-json 0.63, which aws-config still
# requires. Bounds must stay inside 1.x and allow the MSRV job's 1.3.6 pin.
# Drop once aws-config moves to aws-smithy-json 0.64.
aws-smithy-types = { version = ">=1.0, <1.7" }
aws-smithy-types.workspace = true
datafusion.workspace = true
http-body = "1" # Matching reqwest
rstest = "0.23.0"
@@ -130,6 +128,7 @@ pprof = { version = "0.14", features = ["flamegraph"] }
[features]
default = []
aws = [
"dep:aws-smithy-types",
"lance/aws",
"lance-io/aws",
"lance-namespace-impls/dir-aws",
@@ -179,7 +178,7 @@ metrics = ["dep:metrics", "lance/metrics", "lance-io/metrics"]
metrics-otel = ["metrics", "dep:metrics-util"]
fp16kernels = ["lance-linalg/fp16kernels"]
s3-test = []
bedrock = ["dep:aws-sdk-bedrockruntime"]
bedrock = ["dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"]
openai = ["dep:async-openai", "dep:reqwest"]
polars = ["dep:polars-arrow", "dep:polars"]
sentence-transformers = [
+5 -4
View File
@@ -581,10 +581,11 @@ impl Connection {
)
}
/// Register a Python callable as a new immutable Function version.
/// Build and register a Python callable as an immutable Function version.
///
/// Registration is remote-only and always asynchronous. Waiting on the
/// returned typed job yields the durable [`crate::function::FunctionVersion`].
/// The server-side job builds the OCI image, then registers the completed
/// artifact. Waiting on the returned typed job yields the durable
/// [`crate::function::FunctionVersion`]. Creation is remote-only.
/// Local databases return [`Error::NotSupported`].
pub async fn create_function_async(
&self,
@@ -630,7 +631,7 @@ impl Connection {
self.internal.list_functions().await
}
/// Drop one exact immutable Function version from the remote catalog.
/// Remove the current Function name binding, retaining the object history.
///
/// Returns `true` when the server appended a Dropped transition and
/// `false` for an idempotent replay. Local databases return
+2 -2
View File
@@ -296,7 +296,7 @@ pub trait Database:
///
/// See [`CloneTableRequest`] for detailed documentation and examples.
async fn clone_table(&self, request: CloneTableRequest) -> Result<Arc<dyn BaseTable>>;
/// Register an immutable Function version through the remote catalog.
/// Submit a Function creation job that builds an image and registers it.
async fn create_function_async(
&self,
_request: crate::function::FunctionRegistrationRequest,
@@ -382,7 +382,7 @@ pub trait Database:
async fn list_functions(&self) -> Result<Vec<crate::function::FunctionVersion>> {
function_catalog_not_supported()
}
/// Drop one exact immutable Function version from the remote catalog.
/// Remove the current Function name binding, retaining the object history.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
+66 -30
View File
@@ -88,10 +88,18 @@ fn application_has_unknown_nested_fields(value: &Value) -> bool {
let Some(application) = value.as_object() else {
return false;
};
if application
.get("function")
.is_some_and(|value| has_unknown_keys(value, &["name", "version"]))
{
if application.get("function").is_some_and(|value| {
has_unknown_keys(
value,
&[
"name",
"object_id",
"location",
"version",
"manifest_digest",
],
)
}) {
return true;
}
if application
@@ -396,51 +404,75 @@ impl Serialize for PythonRuntimeSpec {
}
}
/// Immutable Function version returned by the Enterprise catalog.
///
/// The GPU execution requirement is part of this identity. CPU and memory sizing,
/// priority, concurrency, and retry policy belong to the execution platform.
/// A complete OCI Function image. Its digest is independent of catalog names.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionImage {
pub manifest_digest: String,
pub descriptor: Value,
pub source: bool,
}
fn deserialize_object_version<'de, D: Deserializer<'de>>(
deserializer: D,
) -> std::result::Result<String, D::Error> {
let value = String::deserialize(deserializer)?;
match value.parse::<u64>() {
Ok(number) if number > 0 && number.to_string() == value => Ok(value),
_ => Err(de::Error::custom(
"Function version must be a canonical positive uint64",
)),
}
}
/// One immutable Function object revision and its executable artifact.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersion {
name: String,
object_id: String,
location: String,
#[serde(deserialize_with = "deserialize_object_version")]
version: String,
artifact: FunctionArtifact,
image: FunctionImage,
signature: FunctionSignature,
runtime: PythonRuntimeSpec,
runtime_digest: String,
environment_digest: String,
created_at: String,
metadata: BTreeMap<String, String>,
disabled: bool,
}
impl FunctionVersion {
pub fn object_id(&self) -> &str {
&self.object_id
}
pub fn location(&self) -> &str {
&self.location
}
pub fn metadata(&self) -> &BTreeMap<String, String> {
&self.metadata
}
pub fn disabled(&self) -> bool {
self.disabled
}
pub fn reference(&self) -> FunctionVersionRef {
FunctionVersionRef {
name: self.name.clone(),
object_id: self.object_id.clone(),
location: self.location.clone(),
version: self.version.clone(),
manifest_digest: self.image.manifest_digest.clone(),
}
}
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> &str {
&self.version
}
pub fn artifact(&self) -> &FunctionArtifact {
&self.artifact
pub fn image(&self) -> &FunctionImage {
&self.image
}
pub fn signature(&self) -> &FunctionSignature {
&self.signature
}
pub fn runtime(&self) -> &PythonRuntimeSpec {
&self.runtime
}
pub fn runtime_digest(&self) -> &str {
&self.runtime_digest
}
pub fn environment_digest(&self) -> &str {
&self.environment_digest
}
pub fn created_at(&self) -> &str {
&self.created_at
}
@@ -496,7 +528,11 @@ impl_json!(FunctionRegistrationRequest);
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionVersionRef {
pub name: String,
pub object_id: String,
pub location: String,
#[serde(deserialize_with = "deserialize_object_version")]
pub version: String,
pub manifest_digest: String,
}
/// Parameter binding in a FunctionApplication.
+4 -2
View File
@@ -2775,7 +2775,8 @@ mod tests {
FunctionBinding::from_json(
&serde_json::json!({
"binding_id": binding_id,
"function": {"name": "embed", "version": "fv_test"},
"function": {"name": "embed", "version": "1", "object_id": "fixture", "location": "memory:///fixture",
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [{
"parameter": "text", "field_id": -1, "field_path": input,
"arrow_type": input_type, "nullable": true,
@@ -3052,7 +3053,8 @@ mod tests {
let binding = FunctionBinding::from_json(
&serde_json::json!({
"binding_id": "fb_pair",
"function": {"name": "pair", "version": "fv_test"},
"function": {"name": "pair", "version": "1", "object_id": "fixture", "location": "memory:///fixture",
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [{"parameter": "value", "field_id": -1, "field_path": "id",
"arrow_type": int, "nullable": true}],
"outputs": [
+9 -9
View File
@@ -3085,7 +3085,7 @@ mod tests {
assert_eq!(job.id(), Some("job-function-1"));
let version = job.wait().await.unwrap();
assert_eq!(version.name(), "embed");
assert_eq!(version.version(), "fv_01K3EXACT");
assert_eq!(version.version(), "1");
}
#[tokio::test]
@@ -3098,12 +3098,12 @@ mod tests {
assert_eq!(request.url().path(), "/v1/function/embed/describe");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(body, serde_json::json!({"version": "fv_01K3EXACT"}));
assert_eq!(body, serde_json::json!({"version": "1"}));
http::Response::builder().status(200).body(VERSION).unwrap()
});
let version = conn.get_function("embed", "fv_01K3EXACT").await.unwrap();
let version = conn.get_function("embed", "1").await.unwrap();
assert_eq!(version.name(), "embed");
assert_eq!(version.version(), "fv_01K3EXACT");
assert_eq!(version.version(), "1");
}
#[tokio::test]
@@ -3134,7 +3134,7 @@ mod tests {
serde_json::json!({
"functions": [{
"name": "embed",
"version": "fv_01K3EXACT",
"version": "1",
"definition": version.clone(),
}],
})
@@ -3147,7 +3147,7 @@ mod tests {
let functions = conn.list_functions().await.unwrap();
assert_eq!(functions.len(), 1);
assert_eq!(functions[0].name(), "embed");
assert_eq!(functions[0].version(), "fv_01K3EXACT");
assert_eq!(functions[0].version(), "1");
}
#[tokio::test]
@@ -3229,13 +3229,13 @@ mod tests {
assert_eq!(request.url().path(), "/v1/function/embed/drop");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(body, serde_json::json!({"version": "fv_01K3EXACT"}));
assert_eq!(body, serde_json::json!({"version": "1"}));
http::Response::builder()
.status(200)
.body(r#"{"dropped":false}"#)
.unwrap()
});
assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap());
assert!(!conn.drop_function("embed", "1").await.unwrap());
}
#[tokio::test]
@@ -3254,7 +3254,7 @@ mod tests {
http::Response::builder()
.status(200)
.body(format!(
r#"{{"job_id": "job-1", "job_type": "create_function", "job_state": "{}", "creation_ms": 1, "result": {{"name": "embed", "version": "fv_1"}}}}"#,
r#"{{"job_id": "job-1", "job_type": "create_function", "job_state": "{}", "creation_ms": 1, "result": {{"name": "embed", "version": "1"}}}}"#,
state
))
.unwrap()
+1 -1
View File
@@ -285,7 +285,7 @@ mod tests {
async fn typed_remote_job_fixtures_decode_terminal_results() {
let function = Job::<FunctionVersion>::new_typed(Box::new(FixtureRemoteJob(FUNCTION_JOB)));
let result = function.wait().await.expect("typed FunctionVersion result");
assert_eq!(result.version(), "fv_01K3EXACT");
assert_eq!(result.version(), "1");
let refresh =
Job::<RefreshColumnResult>::new_typed(Box::new(FixtureRemoteJob(REFRESH_JOB)));
+4 -4
View File
@@ -7856,7 +7856,7 @@ mod tests {
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"embed","version":"fv_01K3EXACT"},
"function":{"name":"embed","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
"output":{"kind":"scalar","arrow_type":"list<float32>","nullable":false}
}"#,
@@ -7933,7 +7933,7 @@ mod tests {
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"text_features","version":"fv_01K3TEXT"},
"function":{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[
{"parameter":"title","kind":"column","value":{"path":"title"}},
{"parameter":"body","kind":"column","value":{"path":"body"}}
@@ -7989,7 +7989,7 @@ mod tests {
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"embed","version":"fv_01K3EXACT"},
"function":{"name":"embed","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
"output":{"kind":"scalar","arrow_type":"fixed_size_list<float32, 3>","nullable":false}
}"#,
@@ -8034,7 +8034,7 @@ mod tests {
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"text_features","version":"fv_01K3TEXT"},
"function":{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[
{"parameter":"title","kind":"column","value":{"path":"title"}},
{"parameter":"body","kind":"column","value":{"path":"body"}}
+17 -11
View File
@@ -539,7 +539,13 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> {
object
.get("function")
.ok_or_else(|| invalid_function("Function binding is missing its exact version"))?,
&["name", "version"],
&[
"name",
"object_id",
"location",
"version",
"manifest_digest",
],
"version reference",
)?;
for input in object
@@ -3022,7 +3028,7 @@ mod tests {
fn named_struct_application(columns: &str) -> FunctionApplication {
FunctionApplication::from_json(&format!(
r#"{{
"function":{{"name":"text_features","version":"fv_exact"}},
"function":{{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}},
"inputs":[
{{"parameter":"title","kind":"column","value":{{"path":"title"}}}},
{{"parameter":"body","kind":"column","value":{{"path":"body"}}}}
@@ -3040,7 +3046,7 @@ mod tests {
fn blob_application(output: &str) -> FunctionApplication {
FunctionApplication::from_json(&format!(
r#"{{
"function":{{"name":"blob_features","version":"fv_blob"}},
"function":{{"name":"blob_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}},
"inputs":[
{{"parameter":"image","kind":"column","value":{{"path":"image"}}}}
],
@@ -3059,7 +3065,7 @@ mod tests {
fn single_input_application(path: &str) -> FunctionApplication {
FunctionApplication::from_json(
&serde_json::json!({
"function": {"name": "inspect", "version": "fv_nested_blob"},
"function": {"name": "inspect", "version": "1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [{
"parameter": "value",
"kind": "column",
@@ -3401,7 +3407,7 @@ mod tests {
));
let dependent_application = FunctionApplication::from_json(
r#"{
"function":{"name":"dependent","version":"fv_dependent"},
"function":{"name":"dependent","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[
{"parameter":"text","kind":"column","value":{"path":"search_text"}}
],
@@ -3532,7 +3538,7 @@ mod tests {
let input = ArrowField::new("value", DataType::Int64, false);
let application = FunctionApplication::from_json(
&serde_json::json!({
"function": {"name": "embed", "version": "fv_embed"},
"function": {"name": "embed", "version": "1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [{
"parameter": "value",
"kind": "column",
@@ -3737,7 +3743,7 @@ mod tests {
));
let application = FunctionApplication::from_json(
&serde_json::json!({
"function": {"name": "inspect", "version": "fv_nested_blob"},
"function": {"name": "inspect", "version": "1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [],
"output": {
"kind": "named_struct",
@@ -3869,7 +3875,7 @@ mod tests {
fn test_unknown_and_mixed_version_function_contracts_fail_closed() {
let application = FunctionApplication::from_json(
r#"{
"function":{"name":"f","version":"fv"},
"function":{"name":"f","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[{"parameter":"title","kind":"future_source","value":{"path":"title"}}],
"output":{"kind":"scalar","arrow_type":"int64","nullable":false}
}"#,
@@ -3881,7 +3887,7 @@ mod tests {
let future_application = FunctionApplication::from_json(
r#"{
"function":{"name":"f","version":"fv"},
"function":{"name":"f","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[],
"output":{"kind":"scalar","arrow_type":"int64","nullable":false},
"future_declaration":{"mode":"managed"}
@@ -3895,7 +3901,7 @@ mod tests {
let nested_future_application = FunctionApplication::from_json(
r#"{
"function":{"name":"f","version":"fv"},
"function":{"name":"f","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[],
"output":{"kind":"scalar","arrow_type":"int64","nullable":false,"assignment":"cell_flag"}
}"#,
@@ -3961,7 +3967,7 @@ mod tests {
let nested_schema = ArrowSchema::new(vec![nested_title, schema.field(1).as_ref().clone()]);
let nested_application = FunctionApplication::from_json(
r#"{
"function":{"name":"text_features","version":"fv_exact"},
"function":{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs":[
{"parameter":"title","kind":"column","value":{"path":"title.value"}},
{"parameter":"body","kind":"column","value":{"path":"body"}}
@@ -26,8 +26,8 @@ fn function_version_job_result_matches_shared_canonical_golden() {
let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
assert_eq!(version.name(), "embed");
assert_eq!(version.version(), "fv_01K3EXACT");
assert_eq!(version.runtime_digest(), "sha256:runtime");
assert_eq!(version.version(), "1");
assert_ne!(version.image().manifest_digest, version.version());
assert_eq!(
version.to_canonical_json().expect("canonical JSON"),
fixture("remote_function_version.canonical.json").trim()
@@ -45,16 +45,28 @@ fn version_identity_is_immutable_and_exact() {
assert_eq!(reopened.version(), version.version());
let mut changed = original;
changed["version"] = Value::String("fv_01K3DIFFERENT".to_string());
changed["version"] = Value::String("2".to_string());
let changed = FunctionVersion::from_json(&changed.to_string()).expect("changed version");
assert_ne!(changed, version);
assert_eq!(changed.image(), version.image());
for invalid in [
version.image().manifest_digest.as_str(),
"0",
"01",
"-1",
"18446744073709551616",
] {
let mut value = serde_json::to_value(&version).unwrap();
value["version"] = Value::String(invalid.into());
assert!(FunctionVersion::from_json(&value.to_string()).is_err());
}
}
#[test]
fn application_and_binding_match_shared_remote_goldens() {
let application = FunctionApplication::from_json(&fixture("remote_function_application.json"))
.expect("application fixture");
assert_eq!(application.function().version, "fv_01K3TEXT");
assert_eq!(application.function().version, "1");
assert_eq!(application.output().kind, "named_struct");
assert_eq!(application.inputs().len(), 2);
assert_eq!(
@@ -64,7 +76,7 @@ fn application_and_binding_match_shared_remote_goldens() {
let binding = FunctionBinding::from_json(&fixture("remote_function_binding.json"))
.expect("binding fixture");
assert_eq!(binding.function().version, "fv_01K3TEXT");
assert_eq!(binding.function().version, "1");
assert_eq!(binding.outputs()[0].output_ordinal, 0);
assert_eq!(binding.outputs()[1].output_ordinal, 1);
assert!(binding.input_schema().is_some());
@@ -113,22 +125,18 @@ fn refresh_job_result_matches_shared_canonical_golden() {
fn unknown_fields_and_discriminators_are_forward_decodable() {
let mut result = job_result("remote_function_job.json");
result["future_version_metadata"] = serde_json::json!({"retention_class": "catalog"});
result["runtime"] = serde_json::json!({
"kind": "wasm",
"module_digest": "sha256:wasm"
});
result["image"]["descriptor"]["future_interface"] = serde_json::json!({"version": 2});
result["signature"]["output"]["kind"] = Value::String("future_output_shape".to_string());
let version = FunctionVersion::from_json(&result.to_string()).expect("future remote value");
assert_eq!(version.runtime().kind(), "wasm");
assert_eq!(version.runtime().python_version(), None);
assert_eq!(version.image().descriptor["future_interface"]["version"], 2);
assert_eq!(version.signature().output.kind, "future_output_shape");
assert_eq!(
serde_json::from_str::<Value>(
&version.to_canonical_json().expect("canonical future value")
)
.expect("canonical JSON")["runtime"],
serde_json::json!({"kind": "wasm"})
.expect("canonical JSON")["image"]["descriptor"]["future_interface"],
serde_json::json!({"version": 2})
);
}
@@ -42,11 +42,11 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
let create_error = connection.create_function_async(request).await.unwrap_err();
let lookup_error = connection
.get_function("normalize_score", "fv_exact")
.get_function("normalize_score", "1")
.await
.unwrap_err();
let drop_error = connection
.drop_function("normalize_score", "fv_exact")
.drop_function("normalize_score", "1")
.await
.unwrap_err();
for error in [create_error, lookup_error, drop_error] {
@@ -9,7 +9,7 @@
"application": {
"function": {
"name": "embed",
"version": "fv_01K3EXACT"
"version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"
},
"inputs": [
{
@@ -1 +1 @@
{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"name":"text_features","version":"fv_01K3TEXT"},"inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}}
{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"text_features","object_id":"fixture","version":"1"},"inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}}
@@ -1,5 +1,5 @@
{
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
"function": {"name": "text_features", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [
{"parameter": "title", "kind": "column", "value": {"path": "title"}},
{"parameter": "body", "kind": "column", "value": {"path": "body"}}
@@ -1,5 +1,5 @@
{
"function": {"name": "score", "version": "fv_01K3FLOAT"},
"function": {"name": "score", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [
{"parameter": "threshold", "kind": "literal", "value": 1e-7}
],
@@ -1 +1 @@
{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}]}
{"binding_id":"fb_01K3TEXT","function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"text_features","object_id":"fixture","version":"1"},"input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}]}
@@ -1,6 +1,6 @@
{
"binding_id": "fb_01K3TEXT",
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
"function": {"name": "text_features", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [
{"parameter": "title", "field_id": 11, "field_path": "title", "arrow_type": "utf8", "nullable": true},
{"parameter": "body", "field_id": 12, "field_path": "body", "arrow_type": "utf8", "nullable": true}
@@ -3,28 +3,64 @@
"job_type": "create_function",
"job_state": "DONE",
"creation_ms": 1787270400000,
"spec": {"name": "embed"},
"spec": {
"name": "embed"
},
"result": {
"name": "embed",
"version": "fv_01K3EXACT",
"artifact": {
"kind": "python_callable",
"digest": "sha256:code",
"entrypoint": "embed"
},
"version": "1", "object_id": "fixture", "location": "memory:///fixture", "metadata": {}, "disabled": false,
"signature": {
"inputs": [{"name": "text", "arrow_type": "utf8", "nullable": true}],
"output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false}
"inputs": [
{
"name": "text",
"arrow_type": "utf8",
"nullable": true
}
],
"output": {
"kind": "scalar",
"arrow_type": "list<float32>",
"nullable": false
}
},
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {"kind": "pip", "packages": ["sentence-transformers>=3"]},
"env": {"TOKENIZERS_PARALLELISM": "false"}
},
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"created_at": "2026-08-21T00:00:00Z"
"created_at": "2026-08-21T00:00:00Z",
"image": {
"manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6",
"descriptor": {
"format_version": 1,
"python": {
"implementation": "cpython",
"version": "3.12.14",
"abi_tag": "cp312",
"executable": "/usr/local/bin/python3",
"import_paths": [
"/opt/function/code"
]
},
"python_api": 1,
"entrypoint": "app.function:create",
"interface": {
"type": "lance.scalar",
"version": 1
},
"schemas": {
"input": "/opt/function/schemas/input.arrow",
"output": "/opt/function/schemas/output.arrow",
"initialization": "/opt/function/schemas/initialization.arrow"
},
"requires": {
"kernel_min": "4.18.0",
"capabilities": []
},
"behavior": {
"result_stability": "input_and_context",
"side_effects": "non_idempotent"
}
},
"source": false
}
},
"future_job": {"trace_id": "trace-1"}
"future_job": {
"trace_id": "trace-1"
}
}
@@ -1 +1 @@
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"1"}
@@ -5,7 +5,7 @@
],
"function": {
"application": {
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
"function": {"name": "text_features", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [
{"parameter": "title", "kind": "column", "value": {"path": "title"}},
{"parameter": "body", "kind": "column", "value": {"path": "body"}}
@@ -4,7 +4,7 @@
],
"function": {
"application": {
"function": {"name": "embed", "version": "fv_01K3EXACT"},
"function": {"name": "embed", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"},
"inputs": [
{"parameter": "text", "kind": "column", "value": {"path": "description"}}
],