diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 74cc2adf4..a5c741cd8 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -495,6 +495,35 @@ class PythonRuntimeSpec(_RemoteValue): object.__setattr__(self, "gpu_class", None) return self + @property + def digest(self) -> str: + """SHA-256 identity of the declared runtime, before building an image. + + Covers Python version, environment sources, environment variables, and + GPU settings. Package and system-package order is ignored; conda channel + order is preserved. Compare with ``runtime_digest`` on + [FunctionVersion][lancedb.functions.FunctionVersion]. + This identifies build inputs, not the resolved contents of unpinned + dependencies or the deployment's default base image. + """ + if self.kind not in {"python", "python_v2"}: + raise ValueError(f"cannot digest unknown runtime kind {self.kind!r}") + value = self._known_dict() + if not value.get("env"): + value.pop("env", None) + environment = value["environment"] + for field in ("packages", "system_packages"): + if field in environment: + environment[field] = sorted(environment[field]) + encoded = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return f"sha256:{hashlib.sha256(encoded).hexdigest()}" + class FunctionImage(_RemoteValue): """A complete OCI Function image identified by its exact manifest digest.""" @@ -536,8 +565,17 @@ class FunctionVersion(_RemoteValue): of the built ``image``. Compare it with ``definition.registration_request.artifact.digest`` to check whether a local [FunctionDefinition][lancedb.functions.FunctionDefinition] has the same source. + It does not cover dependencies, environment variables, Python version, or + GPU settings. Equal source digests alone do not mean a version can be reused. It is ``None`` when the server does not provide a source digest. + ``runtime_digest`` identifies the declared environment inputs. Compare it + with ``definition.registration_request.runtime.digest`` to check whether + the local runtime matches. It is ``None`` for images built before runtime + identity was exposed. A missing digest cannot establish a match. Reusing a + version also requires matching the source and all other registration inputs, + including signature, Secret bindings, models, and execution controls. + ``gpu_class`` is the operator-defined GPU pool fixed into this Function version, or ``None`` when registration used the deployment default. @@ -554,6 +592,7 @@ class FunctionVersion(_RemoteValue): version: _ObjectVersion image: FunctionImage source_digest: Optional[str] = None + runtime_digest: Optional[str] = None signature: FunctionSignature secret_bindings: tuple[SecretBinding, ...] = () created_at_millis: int diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index f8e225183..867672fe9 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -34,7 +34,7 @@ FIXTURES = ( def fixture(name: str) -> str: - return (FIXTURES / name).read_text() + return (FIXTURES / name).read_text(encoding="utf-8") def job_result(name: str) -> dict: @@ -148,6 +148,38 @@ def test_function_version_identity_is_immutable_and_exact(): FunctionVersion(**{**value, "version": invalid}) +@pytest.mark.parametrize( + "case", json.loads(fixture("runtime_identity.json")), ids=lambda case: case["name"] +) +def test_runtime_digests_match_server_golden_values(case): + runtime = PythonRuntimeSpec.from_json(json.dumps(case["runtime"])) + assert runtime.digest == case["digest"] + assert ( + PythonRuntimeSpec.from_json(runtime.to_canonical_json()).digest + == runtime.digest + ) + + +def test_unknown_runtime_digest_fails_closed(): + runtime = PythonRuntimeSpec.from_json('{"kind":"python_v3","future":"value"}') + with pytest.raises(ValueError, match="cannot digest unknown runtime kind"): + runtime.digest + + +@pytest.mark.parametrize("runtime_digest", [None, "sha256:runtime-inputs"]) +def test_function_version_preserves_runtime_digest(runtime_digest): + value = job_result("remote_function_job.json") + legacy = FunctionVersion.from_json(json.dumps(value)) + assert legacy.runtime_digest is None + assert "runtime_digest" not in json.loads(legacy.to_canonical_json()) + value["runtime_digest"] = runtime_digest + version = FunctionVersion.from_json(json.dumps(value)) + assert version.runtime_digest == runtime_digest + assert FunctionVersion.from_json(version.to_canonical_json()) == version + with pytest.raises((TypeError, ValueError)): + version.runtime_digest = "sha256:changed" + + @pytest.mark.parametrize("source_digest", [None, "sha256:source-artifact"]) def test_function_version_preserves_source_digest(source_digest): value = job_result("remote_function_job.json") diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 2cc731892..d99a7aa21 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -1891,6 +1891,29 @@ def test_annotation_and_explicit_schema_validation_fail_closed(): return value +def test_default_function_runtime_matches_server_identity(): + definition = function(python_version="3.12")(analyze_caption.__wrapped__) + runtime = definition.registration_request.runtime + assert runtime.env == {} + cases = json.loads((FIXTURES / "runtime_identity.json").read_text(encoding="utf-8")) + expected = next( + case["digest"] for case in cases if case["name"] == "empty-environment" + ) + assert runtime.digest == expected + + +def test_environment_changes_do_not_change_source_identity(): + definition = function( + pip=["openai==3.46.0"], + env={"MODE": "test"}, + python_version="3.12", + )(analyze_caption.__wrapped__) + previous = analyze_caption.registration_request + changed = definition.registration_request + assert previous.artifact.digest == changed.artifact.digest + assert previous.runtime.digest != changed.runtime.digest + + def test_local_function_catalog_operations_are_not_supported(tmp_path): db = lancedb.connect(tmp_path) message = "Function catalog operations are not supported by this database" @@ -1950,6 +1973,7 @@ async def test_async_catalog_path_preserves_execution_controls(): (FIXTURES / "remote_function_version.canonical.json").read_text() ) definition.update( + runtime_digest=normalize_score.registration_request.runtime.digest, batch_size=12, row_timeout_seconds=8.0, result_stability="input_and_context", @@ -1977,6 +2001,7 @@ def test_sync_catalog_path_preserves_execution_controls(): (FIXTURES / "remote_function_version.canonical.json").read_text() ) definition.update( + runtime_digest=normalize_score.registration_request.runtime.digest, batch_size=12, row_timeout_seconds=8.0, result_stability="input_and_context", @@ -2090,6 +2115,9 @@ def _mock_remote_function_catalog(): )["image"], "signature": body["signature"], "source_digest": body["artifact"]["digest"], + "runtime_digest": PythonRuntimeSpec.from_json( + json.dumps(body["runtime"]) + ).digest, "secret_bindings": body.get("secret_bindings", []), "created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z } diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 952437051..b0098aca3 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -6304,6 +6304,8 @@ def test_remote_function_registration_transports_max_concurrency(): def registered(value: float) -> float: return value + definition["runtime_digest"] = registered.registration_request.runtime.digest + class RegistrationServer(flight.FlightServerBase): def __init__(self): super().__init__("grpc://127.0.0.1:0") @@ -6322,6 +6324,7 @@ def test_remote_function_registration_transports_max_concurrency(): assert len(server.tickets) == 1 assert b'"max_concurrency":4' in server.tickets[0] assert version.max_concurrency == 4 + assert version.runtime_digest == registered.registration_request.runtime.digest @pytest.mark.asyncio diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 3566fc938..e3d47948e 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -594,6 +594,8 @@ pub struct FunctionVersion { image: FunctionImage, #[serde(default, skip_serializing_if = "Option::is_none")] source_digest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + runtime_digest: Option, signature: FunctionSignature, #[serde(default, skip_serializing_if = "Vec::is_empty")] secret_bindings: Vec, @@ -665,7 +667,9 @@ impl FunctionVersion { /// Digest of the registered source artifact, when provided by the server. /// /// Compare this with [`FunctionArtifactRequest::digest`] to identify matching - /// source, independently of the built image's manifest digest. + /// source, independently of the built image's manifest digest. It excludes + /// dependencies, environment variables, Python version, and GPU settings; + /// matching source alone does not establish that a version can be reused. /// /// ``` /// use lancedb::function::{FunctionArtifactRequest, FunctionVersion}; @@ -678,6 +682,15 @@ impl FunctionVersion { self.source_digest.as_deref() } + /// Digest of canonical runtime inputs, when provided by the server. + /// + /// Includes Python version, environment sources, variables, and GPU settings. + /// A missing digest cannot establish that a local runtime matches. Reusing a + /// version also requires matching its source and other registration inputs. + pub fn runtime_digest(&self) -> Option<&str> { + self.runtime_digest.as_deref() + } + /// Operator-defined GPU class declared at registration. pub fn gpu_class(&self) -> Option<&str> { self.gpu_class.as_deref() @@ -1329,6 +1342,30 @@ mod conda_environment_tests { mod tests { use super::*; + #[test] + fn function_version_preserves_runtime_digest() { + const VERSION: &str = include_str!( + "../tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json" + ); + let mut document: Value = serde_json::from_str(VERSION).unwrap(); + let legacy = FunctionVersion::from_json(VERSION).unwrap(); + assert_eq!(legacy.runtime_digest(), None); + assert_eq!(legacy.to_canonical_json().unwrap(), VERSION.trim_end()); + for digest in [Value::Null, serde_json::json!("sha256:runtime-inputs")] { + document["runtime_digest"] = digest.clone(); + let version = FunctionVersion::from_json(&document.to_string()).unwrap(); + assert_eq!(version.runtime_digest(), digest.as_str()); + let encoded = version.to_canonical_json().unwrap(); + let encoded_value: Value = serde_json::from_str(&encoded).unwrap(); + if digest.is_null() { + assert!(encoded_value.get("runtime_digest").is_none()); + } else { + assert_eq!(encoded_value["runtime_digest"], digest); + } + assert_eq!(FunctionVersion::from_json(&encoded).unwrap(), version); + } + } + #[test] fn function_version_preserves_source_digest() { const VERSION: &str = include_str!( diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/runtime_identity.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/runtime_identity.json new file mode 100644 index 000000000..7bf0c375c --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/runtime_identity.json @@ -0,0 +1,269 @@ +[ + { + "name": "pip", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:e2891a0af2567af2798c7de67a2f91b031088ef2a7015b501022892995eaca1c" + }, + { + "name": "reordered-packages", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "numpy==2.2.0", + "openai==3.40.0" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:e2891a0af2567af2798c7de67a2f91b031088ef2a7015b501022892995eaca1c" + }, + { + "name": "changed-pin", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.46.0", + "numpy==2.2.0" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:43da49cfcf8ecd47becb5be20b9eff06c5267f9244c18e751b230c624f5ad12a" + }, + { + "name": "changed-env", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ] + }, + "env": { + "MODE": "prod", + "LABEL": "你好" + } + }, + "digest": "sha256:9b305e9188f4ddda5a0b72d2235c2c46ba50de9bf82c699f5060b02e4aa0820d" + }, + { + "name": "changed-python", + "runtime": { + "kind": "python", + "python_version": "3.11", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:8fa58561581414b02d0619e466e3720828d1820d8211b0e73c90ec4b0d3ecd0c" + }, + { + "name": "gpu", + "runtime": { + "kind": "python_v2", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + }, + "gpu": true + }, + "digest": "sha256:3ea9354ff101038937b1f5a3b6639c4cc6dcb88ee975dcb8e1a5dee2500c283c" + }, + { + "name": "gpu-class", + "runtime": { + "kind": "python_v2", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + }, + "gpu": true, + "gpu_class": "a100" + }, + "digest": "sha256:253f0641ef029aed8337bbcb210f7769147d671747406f7ce4fa5dc40d5c5efc" + }, + { + "name": "conda", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "conda", + "packages": [ + "numpy=2.2.0", + "scipy=1.15.0" + ], + "channels": [ + "conda-forge", + "defaults" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:9246f88a6b1daceb3090c51930650ea967947142cdc70737c847a6bf72c85d30" + }, + { + "name": "reordered-channels", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "conda", + "packages": [ + "numpy=2.2.0", + "scipy=1.15.0" + ], + "channels": [ + "defaults", + "conda-forge" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:12ef13a6f5b8730a719288ab2406e7f6c13634b14dbef2390f0bbe62f847e18d" + }, + { + "name": "system-packages", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ], + "system_packages": [ + "zlib1g", + "ffmpeg" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:129706ec54d8bf5fe36f89a9cf9131b37b2293028e03da7d686c77c3c483d958" + }, + { + "name": "reordered-system-packages", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [ + "openai==3.40.0", + "numpy==2.2.0" + ], + "system_packages": [ + "ffmpeg", + "zlib1g" + ] + }, + "env": { + "MODE": "test", + "LABEL": "你好" + } + }, + "digest": "sha256:129706ec54d8bf5fe36f89a9cf9131b37b2293028e03da7d686c77c3c483d958" + }, + { + "name": "empty-environment", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip" + } + }, + "digest": "sha256:a3d52daa7573e7fc6d052e0bf05b5189b9413ad8e136c10bc48f6dedc3add047" + }, + { + "name": "explicit-empty-env", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip" + }, + "env": {} + }, + "digest": "sha256:a3d52daa7573e7fc6d052e0bf05b5189b9413ad8e136c10bc48f6dedc3add047" + }, + { + "name": "explicit-empty-fields", + "runtime": { + "kind": "python", + "python_version": "3.12", + "environment": { + "kind": "pip", + "packages": [], + "channels": [], + "modules": [], + "system_packages": [] + }, + "env": {} + }, + "digest": "sha256:a3d52daa7573e7fc6d052e0bf05b5189b9413ad8e136c10bc48f6dedc3add047" + } +]