rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: expose comparable Function runtime identity (#4975)

Browse Source 
Expose `FunctionVersion.runtime_digest` through Rust and Python, and add
`PythonRuntimeSpec.digest` so clients can compare declared environment
inputs before registering a Function. Dependency pins, environment
variables, Python version, and GPU settings affect this digest; package
ordering does not.

Clarify that matching `source_digest` alone is insufficient for reuse.
Older servers/images leave `runtime_digest` unset, which cannot
establish a match. Both digests and the other registration inputs must
match before reusing a version.

```python
request = definition.registration_request
same_source = version.source_digest == request.artifact.digest
same_runtime = version.runtime_digest == request.runtime.digest
```

Add shared runtime digest fixtures and coverage for environment-only
changes and catalog decoding. Local builds and tests were skipped as
requested.
This commit is contained in:
Jack Ye
2026-11-12 00:30:19 -08:00
committed by GitHub
parent 01e9ea44aa
commit 478d1d9b01
6 changed files with 410 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/functions.py
+39
View File
@@ -495,6 +495,35 @@ class PythonRuntimeSpec(_RemoteValue):
object.__setattr__(self, "gpu_class", None)
return self
@property
def digest(self) -> str:
"""SHA-256 identity of the declared runtime, before building an image.
Covers Python version, environment sources, environment variables, and
GPU settings. Package and system-package order is ignored; conda channel
order is preserved. Compare with ``runtime_digest`` on
[FunctionVersion][lancedb.functions.FunctionVersion].
This identifies build inputs, not the resolved contents of unpinned
dependencies or the deployment's default base image.
"""
if self.kind not in {"python", "python_v2"}:
raise ValueError(f"cannot digest unknown runtime kind {self.kind!r}")
value = self._known_dict()
if not value.get("env"):
value.pop("env", None)
environment = value["environment"]
for field in ("packages", "system_packages"):
if field in environment:
environment[field] = sorted(environment[field])
encoded = json.dumps(
value,
ensure_ascii=False,
allow_nan=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
class FunctionImage(_RemoteValue):
"""A complete OCI Function image identified by its exact manifest digest."""
@@ -536,8 +565,17 @@ class FunctionVersion(_RemoteValue):
of the built ``image``. Compare it with
``definition.registration_request.artifact.digest`` to check whether a local
[FunctionDefinition][lancedb.functions.FunctionDefinition] has the same source.
It does not cover dependencies, environment variables, Python version, or
GPU settings. Equal source digests alone do not mean a version can be reused.
It is ``None`` when the server does not provide a source digest.
``runtime_digest`` identifies the declared environment inputs. Compare it
with ``definition.registration_request.runtime.digest`` to check whether
the local runtime matches. It is ``None`` for images built before runtime
identity was exposed. A missing digest cannot establish a match. Reusing a
version also requires matching the source and all other registration inputs,
including signature, Secret bindings, models, and execution controls.
``gpu_class`` is the operator-defined GPU pool fixed into this Function
version, or ``None`` when registration used the deployment default.
@@ -554,6 +592,7 @@ class FunctionVersion(_RemoteValue):
version: _ObjectVersion
image: FunctionImage
source_digest: Optional[str] = None
runtime_digest: Optional[str] = None
signature: FunctionSignature
secret_bindings: tuple[SecretBinding, ...] = ()
created_at_millis: int
python/python/tests/test_first_class_function_slice1.py
+33 -1
View File
@@ -34,7 +34,7 @@ FIXTURES = (
def fixture(name: str) -> str:
return (FIXTURES / name).read_text()
return (FIXTURES / name).read_text(encoding="utf-8")
def job_result(name: str) -> dict:
@@ -148,6 +148,38 @@ def test_function_version_identity_is_immutable_and_exact():
FunctionVersion(**{**value, "version": invalid})
@pytest.mark.parametrize(
"case", json.loads(fixture("runtime_identity.json")), ids=lambda case: case["name"]
)
def test_runtime_digests_match_server_golden_values(case):
runtime = PythonRuntimeSpec.from_json(json.dumps(case["runtime"]))
assert runtime.digest == case["digest"]
assert (
PythonRuntimeSpec.from_json(runtime.to_canonical_json()).digest
== runtime.digest
)
def test_unknown_runtime_digest_fails_closed():
runtime = PythonRuntimeSpec.from_json('{"kind":"python_v3","future":"value"}')
with pytest.raises(ValueError, match="cannot digest unknown runtime kind"):
runtime.digest
@pytest.mark.parametrize("runtime_digest", [None, "sha256:runtime-inputs"])
def test_function_version_preserves_runtime_digest(runtime_digest):
value = job_result("remote_function_job.json")
legacy = FunctionVersion.from_json(json.dumps(value))
assert legacy.runtime_digest is None
assert "runtime_digest" not in json.loads(legacy.to_canonical_json())
value["runtime_digest"] = runtime_digest
version = FunctionVersion.from_json(json.dumps(value))
assert version.runtime_digest == runtime_digest
assert FunctionVersion.from_json(version.to_canonical_json()) == version
with pytest.raises((TypeError, ValueError)):
version.runtime_digest = "sha256:changed"
@pytest.mark.parametrize("source_digest", [None, "sha256:source-artifact"])
def test_function_version_preserves_source_digest(source_digest):
value = job_result("remote_function_job.json")
python/python/tests/test_first_class_function_slice2.py
+28
View File
@@ -1891,6 +1891,29 @@ def test_annotation_and_explicit_schema_validation_fail_closed():
return value
def test_default_function_runtime_matches_server_identity():
definition = function(python_version="3.12")(analyze_caption.__wrapped__)
runtime = definition.registration_request.runtime
assert runtime.env == {}
cases = json.loads((FIXTURES / "runtime_identity.json").read_text(encoding="utf-8"))
expected = next(
case["digest"] for case in cases if case["name"] == "empty-environment"
)
assert runtime.digest == expected
def test_environment_changes_do_not_change_source_identity():
definition = function(
pip=["openai==3.46.0"],
env={"MODE": "test"},
python_version="3.12",
)(analyze_caption.__wrapped__)
previous = analyze_caption.registration_request
changed = definition.registration_request
assert previous.artifact.digest == changed.artifact.digest
assert previous.runtime.digest != changed.runtime.digest
def test_local_function_catalog_operations_are_not_supported(tmp_path):
db = lancedb.connect(tmp_path)
message = "Function catalog operations are not supported by this database"
@@ -1950,6 +1973,7 @@ async def test_async_catalog_path_preserves_execution_controls():
(FIXTURES / "remote_function_version.canonical.json").read_text()
)
definition.update(
runtime_digest=normalize_score.registration_request.runtime.digest,
batch_size=12,
row_timeout_seconds=8.0,
result_stability="input_and_context",
@@ -1977,6 +2001,7 @@ def test_sync_catalog_path_preserves_execution_controls():
(FIXTURES / "remote_function_version.canonical.json").read_text()
)
definition.update(
runtime_digest=normalize_score.registration_request.runtime.digest,
batch_size=12,
row_timeout_seconds=8.0,
result_stability="input_and_context",
@@ -2090,6 +2115,9 @@ def _mock_remote_function_catalog():
)["image"],
"signature": body["signature"],
"source_digest": body["artifact"]["digest"],
"runtime_digest": PythonRuntimeSpec.from_json(
json.dumps(body["runtime"])
).digest,
"secret_bindings": body.get("secret_bindings", []),
"created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z
}
python/python/tests/test_remote_db.py
+3
View File
@@ -6304,6 +6304,8 @@ def test_remote_function_registration_transports_max_concurrency():
def registered(value: float) -> float:
return value
definition["runtime_digest"] = registered.registration_request.runtime.digest
class RegistrationServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
@@ -6322,6 +6324,7 @@ def test_remote_function_registration_transports_max_concurrency():
assert len(server.tickets) == 1
assert b'"max_concurrency":4' in server.tickets[0]
assert version.max_concurrency == 4
assert version.runtime_digest == registered.registration_request.runtime.digest
@pytest.mark.asyncio
rust/lancedb/src/function.rs
+38 -1
View File
@@ -594,6 +594,8 @@ pub struct FunctionVersion {
image: FunctionImage,
#[serde(default, skip_serializing_if = "Option::is_none")]
source_digest: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
runtime_digest: Option<String>,
signature: FunctionSignature,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
secret_bindings: Vec<SecretBinding>,
@@ -665,7 +667,9 @@ impl FunctionVersion {
/// Digest of the registered source artifact, when provided by the server.
///
/// Compare this with [`FunctionArtifactRequest::digest`] to identify matching
/// source, independently of the built image's manifest digest.
/// source, independently of the built image's manifest digest. It excludes
/// dependencies, environment variables, Python version, and GPU settings;
/// matching source alone does not establish that a version can be reused.
///
/// ```
/// use lancedb::function::{FunctionArtifactRequest, FunctionVersion};
@@ -678,6 +682,15 @@ impl FunctionVersion {
self.source_digest.as_deref()
}
/// Digest of canonical runtime inputs, when provided by the server.
///
/// Includes Python version, environment sources, variables, and GPU settings.
/// A missing digest cannot establish that a local runtime matches. Reusing a
/// version also requires matching its source and other registration inputs.
pub fn runtime_digest(&self) -> Option<&str> {
self.runtime_digest.as_deref()
}
/// Operator-defined GPU class declared at registration.
pub fn gpu_class(&self) -> Option<&str> {
self.gpu_class.as_deref()
@@ -1329,6 +1342,30 @@ mod conda_environment_tests {
mod tests {
use super::*;
#[test]
fn function_version_preserves_runtime_digest() {
const VERSION: &str = include_str!(
"../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json"
);
let mut document: Value = serde_json::from_str(VERSION).unwrap();
let legacy = FunctionVersion::from_json(VERSION).unwrap();
assert_eq!(legacy.runtime_digest(), None);
assert_eq!(legacy.to_canonical_json().unwrap(), VERSION.trim_end());
for digest in [Value::Null, serde_json::json!("sha256:runtime-inputs")] {
document["runtime_digest"] = digest.clone();
let version = FunctionVersion::from_json(&document.to_string()).unwrap();
assert_eq!(version.runtime_digest(), digest.as_str());
let encoded = version.to_canonical_json().unwrap();
let encoded_value: Value = serde_json::from_str(&encoded).unwrap();
if digest.is_null() {
assert!(encoded_value.get("runtime_digest").is_none());
} else {
assert_eq!(encoded_value["runtime_digest"], digest);
}
assert_eq!(FunctionVersion::from_json(&encoded).unwrap(), version);
}
}
#[test]
fn function_version_preserves_source_digest() {
const VERSION: &str = include_str!(
rust/lancedb/tests/fixtures/first_class_functions/v1/runtime_identity.json
Vendored
+269
View File
@@ -0,0 +1,269 @@
[
{
"name": "pip",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:e2891a0af2567af2798c7de67a2f91b031088ef2a7015b501022892995eaca1c"
},
{
"name": "reordered-packages",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"numpy==2.2.0",
"openai==3.40.0"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:e2891a0af2567af2798c7de67a2f91b031088ef2a7015b501022892995eaca1c"
},
{
"name": "changed-pin",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.46.0",
"numpy==2.2.0"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:43da49cfcf8ecd47becb5be20b9eff06c5267f9244c18e751b230c624f5ad12a"
},
{
"name": "changed-env",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
]
},
"env": {
"MODE": "prod",
"LABEL": "你好"
}
},
"digest": "sha256:9b305e9188f4ddda5a0b72d2235c2c46ba50de9bf82c699f5060b02e4aa0820d"
},
{
"name": "changed-python",
"runtime": {
"kind": "python",
"python_version": "3.11",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:8fa58561581414b02d0619e466e3720828d1820d8211b0e73c90ec4b0d3ecd0c"
},
{
"name": "gpu",
"runtime": {
"kind": "python_v2",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
},
"gpu": true
},
"digest": "sha256:3ea9354ff101038937b1f5a3b6639c4cc6dcb88ee975dcb8e1a5dee2500c283c"
},
{
"name": "gpu-class",
"runtime": {
"kind": "python_v2",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
},
"gpu": true,
"gpu_class": "a100"
},
"digest": "sha256:253f0641ef029aed8337bbcb210f7769147d671747406f7ce4fa5dc40d5c5efc"
},
{
"name": "conda",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "conda",
"packages": [
"numpy=2.2.0",
"scipy=1.15.0"
],
"channels": [
"conda-forge",
"defaults"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:9246f88a6b1daceb3090c51930650ea967947142cdc70737c847a6bf72c85d30"
},
{
"name": "reordered-channels",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "conda",
"packages": [
"numpy=2.2.0",
"scipy=1.15.0"
],
"channels": [
"defaults",
"conda-forge"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:12ef13a6f5b8730a719288ab2406e7f6c13634b14dbef2390f0bbe62f847e18d"
},
{
"name": "system-packages",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
],
"system_packages": [
"zlib1g",
"ffmpeg"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:129706ec54d8bf5fe36f89a9cf9131b37b2293028e03da7d686c77c3c483d958"
},
{
"name": "reordered-system-packages",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [
"openai==3.40.0",
"numpy==2.2.0"
],
"system_packages": [
"ffmpeg",
"zlib1g"
]
},
"env": {
"MODE": "test",
"LABEL": "你好"
}
},
"digest": "sha256:129706ec54d8bf5fe36f89a9cf9131b37b2293028e03da7d686c77c3c483d958"
},
{
"name": "empty-environment",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip"
}
},
"digest": "sha256:a3d52daa7573e7fc6d052e0bf05b5189b9413ad8e136c10bc48f6dedc3add047"
},
{
"name": "explicit-empty-env",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip"
},
"env": {}
},
"digest": "sha256:a3d52daa7573e7fc6d052e0bf05b5189b9413ad8e136c10bc48f6dedc3add047"
},
{
"name": "explicit-empty-fields",
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {
"kind": "pip",
"packages": [],
"channels": [],
"modules": [],
"system_packages": []
},
"env": {}
},
"digest": "sha256:a3d52daa7573e7fc6d052e0bf05b5189b9413ad8e136c10bc48f6dedc3add047"
}
]