mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(function): expose resolved environments and fresh lock requests (#5034)
Function registration clients cannot request fresh dependency resolution or inspect the measured environment returned by the server. Add `refresh_lock=True` to Python registration and preserve the requested runtime and measured environments in Rust/Python Function version readback. Registration forwards both `exist_ok` and `refresh_lock`; defaults remain unchanged. Single-platform versions retain `resolved_environment`. Multiarchitecture versions expose `resolved_environments`, keyed by OCI `os/architecture`, and the Python `environment_report` alias. Platforms are never flattened: absent entries mean unknown measurements, while measured empty package lists remain present. Registration waits emit scalar and per-platform server warnings once per Job; catalog reads do not repeat them. Requires matching server support, including Sophon #10344 for per-platform publication. Nine focused Python source-overlay tests and 29 documentation tests passed for the additive map change; one documentation test requiring a newer native `exist_ok` binding was excluded. These checks reuse a prebuilt extension and do not establish current native integration. PR CI exercises the Rust transport/readback and Python bindings. Live multiarchitecture registration and GPU acceptance remain separate.
This commit is contained in:
@@ -51,11 +51,15 @@ from .functions import (
|
||||
FunctionApplication as FunctionApplication,
|
||||
FunctionBinding as FunctionBinding,
|
||||
FunctionDefinition as FunctionDefinition,
|
||||
FunctionEnvironment as FunctionEnvironment,
|
||||
FunctionRegistrationRequest as FunctionRegistrationRequest,
|
||||
FunctionRetryPolicy as FunctionRetryPolicy,
|
||||
FunctionVersion as FunctionVersion,
|
||||
HuggingFaceModel as HuggingFaceModel,
|
||||
PythonRuntimeSpec as PythonRuntimeSpec,
|
||||
ResolvedEnvironment as ResolvedEnvironment,
|
||||
ResolvedEnvironmentProvenance as ResolvedEnvironmentProvenance,
|
||||
ResolvedPackage as ResolvedPackage,
|
||||
RefreshColumnResult as RefreshColumnResult,
|
||||
function as function,
|
||||
udf as udf,
|
||||
|
||||
@@ -154,6 +154,38 @@ def _normalize_function_version(version: Optional[Union[int, str]]) -> Optional[
|
||||
raise TypeError("version must be an int, str, or None")
|
||||
|
||||
|
||||
def _function_registration_decoder():
|
||||
"""Parse a registration result and emit its server warnings once.
|
||||
|
||||
The warnings come from the scalar environment or, for a multiarchitecture
|
||||
image, each measured platform entry. They are part of the registration job
|
||||
result, not a new client-side package policy. Catalog get/list readbacks
|
||||
use ``FunctionVersion.from_json`` directly and therefore do not replay
|
||||
them.
|
||||
"""
|
||||
warned = False
|
||||
|
||||
def decode(payload: str) -> FunctionVersion:
|
||||
nonlocal warned
|
||||
version = FunctionVersion.from_json(payload)
|
||||
if not warned:
|
||||
environments = [
|
||||
environment
|
||||
for environment in (
|
||||
version.resolved_environment,
|
||||
*version.resolved_environments.values(),
|
||||
)
|
||||
if environment is not None
|
||||
]
|
||||
for environment in environments:
|
||||
for message in environment.warnings:
|
||||
warnings.warn(message, UserWarning, stacklevel=3)
|
||||
warned = True
|
||||
return version
|
||||
|
||||
return decode
|
||||
|
||||
|
||||
class DBConnection(EnforceOverrides):
|
||||
"""An active LanceDB connection interface.
|
||||
|
||||
@@ -1018,6 +1050,7 @@ class DBConnection(EnforceOverrides):
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
refresh_lock: bool = False,
|
||||
) -> FunctionVersion:
|
||||
"""Build and register a scalar Python function, then return its version.
|
||||
|
||||
@@ -1040,6 +1073,13 @@ class DBConnection(EnforceOverrides):
|
||||
namespace_path : list of str, optional
|
||||
The namespace holding the Function. None or an empty list is the
|
||||
root namespace.
|
||||
refresh_lock : bool, default False
|
||||
Ask the server to resolve a fresh environment lock instead of
|
||||
reusing the published version's lock.
|
||||
|
||||
When the registration job resolves, each resolved-environment warning
|
||||
reported by the server is emitted as a Python ``UserWarning``. Catalog
|
||||
reads do not replay those registration warnings.
|
||||
|
||||
Examples
|
||||
--------
|
||||
@@ -1060,6 +1100,7 @@ class DBConnection(EnforceOverrides):
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
refresh_lock=refresh_lock,
|
||||
).wait()
|
||||
|
||||
def create_function_async(
|
||||
@@ -1069,6 +1110,7 @@ class DBConnection(EnforceOverrides):
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
refresh_lock: bool = False,
|
||||
) -> Job[FunctionVersion]:
|
||||
"""Submit a scalar Python function for building and registration.
|
||||
|
||||
@@ -1079,6 +1121,11 @@ class DBConnection(EnforceOverrides):
|
||||
artifact. The returned job has type ``query`` and the SQL query UUID as
|
||||
its ID. Waiting on it returns the immutable Function version.
|
||||
LanceDB Embedded connections raise ``NotImplementedError``.
|
||||
|
||||
``refresh_lock`` asks the server to resolve a fresh environment lock
|
||||
instead of reusing the published version's lock. Waiting on the job
|
||||
emits the server's resolved-environment warnings as Python warnings;
|
||||
reading the same version later does not repeat them.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
_append_doc_url(
|
||||
@@ -2432,6 +2479,7 @@ class LanceDBConnection(DBConnection):
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
refresh_lock: bool = False,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(
|
||||
self._conn.create_function_async(
|
||||
@@ -2439,6 +2487,7 @@ class LanceDBConnection(DBConnection):
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
refresh_lock=refresh_lock,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
@@ -3788,6 +3837,7 @@ class AsyncConnection(object):
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
refresh_lock: bool = False,
|
||||
) -> AsyncJob[FunctionVersion]:
|
||||
"""Submit a scalar Python function for building and registration.
|
||||
|
||||
@@ -3802,14 +3852,16 @@ class AsyncConnection(object):
|
||||
the environment variable its value arrives in. ``namespace_path`` is
|
||||
the namespace the Function is registered in; None or an empty list is
|
||||
the root. LanceDB Embedded connections raise ``NotImplementedError``.
|
||||
``refresh_lock`` asks the server to resolve a fresh environment lock
|
||||
instead of reusing the published version's lock.
|
||||
"""
|
||||
if not isinstance(definition, FunctionDefinition):
|
||||
raise TypeError("create_function_async requires a @function definition")
|
||||
request = definition.bind_secrets(secrets)
|
||||
request = definition.bind_secrets(secrets).with_refresh_lock(refresh_lock)
|
||||
inner = await self._inner.create_function_async(
|
||||
request.to_canonical_json(), namespace_path, exist_ok=exist_ok
|
||||
)
|
||||
return _typed_job(inner, FunctionVersion.from_json)
|
||||
return _typed_job(inner, _function_registration_decoder())
|
||||
|
||||
async def get_function(
|
||||
self,
|
||||
|
||||
@@ -36,6 +36,7 @@ from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Literal,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
Union,
|
||||
@@ -101,6 +102,12 @@ def _validate_batch_size(value: Any) -> Optional[int]:
|
||||
return value
|
||||
|
||||
|
||||
def _validate_refresh_lock(value: Any) -> bool:
|
||||
if not isinstance(value, bool):
|
||||
raise ValueError("refresh_lock must be a boolean")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_max_concurrency(value: Any) -> Optional[int]:
|
||||
if value is None:
|
||||
return None
|
||||
@@ -544,6 +551,67 @@ class FunctionImage(_RemoteValue):
|
||||
models: tuple[HuggingFaceModel, ...] = ()
|
||||
|
||||
|
||||
class ResolvedPackage(_RemoteValue):
|
||||
"""One installed Python distribution measured from a baked environment.
|
||||
|
||||
``version`` is the complete version reported by package metadata,
|
||||
including local labels such as ``+cu130``. ``source`` is present only when
|
||||
the submitted lock retained a per-package source; absent means unknown, not
|
||||
an inferred index.
|
||||
"""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
source: Optional[str] = None
|
||||
|
||||
|
||||
class ResolvedEnvironmentProvenance(_RemoteValue):
|
||||
"""Sanitized source and accelerator provenance for a resolved environment.
|
||||
|
||||
``configured_index_urls`` and ``configured_find_links`` record build
|
||||
inputs. ``resolved_indexes`` records indexes visible in the submitted lock,
|
||||
so a configured source is not by itself evidence that a package came from
|
||||
it. Empty lists and absent fields remain absent; the client does not
|
||||
synthesize them.
|
||||
"""
|
||||
|
||||
configured_index_urls: tuple[str, ...] = ()
|
||||
configured_find_links: tuple[str, ...] = ()
|
||||
resolved_indexes: tuple[str, ...] = ()
|
||||
cuda_toolkit: Optional[str] = None
|
||||
torch_version: Optional[str] = None
|
||||
|
||||
|
||||
class ResolvedEnvironment(_RemoteValue):
|
||||
"""Actual installed distributions and registration warnings for one platform.
|
||||
|
||||
A missing value means its measurement is unknown, not an empty
|
||||
installed-package list. Versions are retained verbatim, including local
|
||||
labels; package ``source`` is retained only when the producer recorded it.
|
||||
"""
|
||||
|
||||
packages: tuple[ResolvedPackage, ...]
|
||||
provenance: Optional[ResolvedEnvironmentProvenance] = None
|
||||
warnings: tuple[str, ...] = ()
|
||||
|
||||
|
||||
class FunctionEnvironment(NamedTuple):
|
||||
"""Requested and measured environment view for a Function version.
|
||||
|
||||
This is a view over
|
||||
[FunctionVersion.runtime][lancedb.functions.FunctionVersion.runtime]
|
||||
and
|
||||
[FunctionVersion.resolved_environment][lancedb.functions.FunctionVersion.resolved_environment].
|
||||
It adds no wire field and performs no second parse. ``requested`` is the
|
||||
runtime specification supplied at registration; ``resolved`` is measured
|
||||
data and is ``None`` when unknown. For per-platform multiarchitecture
|
||||
measurements, use ``FunctionVersion.environment_report``.
|
||||
"""
|
||||
|
||||
requested: Optional[PythonRuntimeSpec]
|
||||
resolved: Optional[ResolvedEnvironment]
|
||||
|
||||
|
||||
def _validate_object_version(value: str) -> str:
|
||||
if int(value) > 2**64 - 1:
|
||||
raise ValueError("Function version exceeds uint64")
|
||||
@@ -593,6 +661,17 @@ class FunctionVersion(_RemoteValue):
|
||||
``result_stability`` retain the execution controls reported by the
|
||||
catalog. They are ``None`` for versions registered before the
|
||||
corresponding declaration.
|
||||
|
||||
``runtime`` is the runtime specification requested at registration. It is
|
||||
not evidence of what was installed. ``resolved_environment`` is measured
|
||||
data for a single-platform image: actual package names and complete
|
||||
versions, optional per-package source, provenance, and registration
|
||||
warnings. ``resolved_environments`` is the corresponding map for a
|
||||
multiarchitecture image, keyed by OCI ``os/architecture``. It is never
|
||||
flattened into an aggregate package list. ``None`` or an empty map means
|
||||
unknown data, not an empty installed environment. ``environment`` is a
|
||||
convenience view with ``requested`` and ``resolved`` members over the same
|
||||
fields.
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -613,6 +692,11 @@ class FunctionVersion(_RemoteValue):
|
||||
max_concurrency: Optional[int] = None
|
||||
row_timeout_seconds: Optional[float] = None
|
||||
result_stability: Optional[str] = None
|
||||
runtime: Optional[PythonRuntimeSpec] = None
|
||||
resolved_environment: Optional[ResolvedEnvironment] = None
|
||||
resolved_environments: Mapping[str, ResolvedEnvironment] = Field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
@field_validator("batch_size", mode="before")
|
||||
@classmethod
|
||||
@@ -629,6 +713,24 @@ class FunctionVersion(_RemoteValue):
|
||||
def _validate_row_timeout(cls, value):
|
||||
return _validate_row_timeout(value)
|
||||
|
||||
@property
|
||||
def environment(self) -> FunctionEnvironment:
|
||||
"""Requested and measured environment without adding a second parser."""
|
||||
return FunctionEnvironment(
|
||||
requested=self.runtime,
|
||||
resolved=self.resolved_environment,
|
||||
)
|
||||
|
||||
@property
|
||||
def environment_report(self) -> Mapping[str, ResolvedEnvironment]:
|
||||
"""Per-platform measured reports, keyed by OCI ``os/architecture``.
|
||||
|
||||
This is the same map as ``resolved_environments`` and is never
|
||||
flattened. Single-platform images use ``resolved_environment`` instead;
|
||||
an absent platform means its measurement is unknown.
|
||||
"""
|
||||
return self.resolved_environments
|
||||
|
||||
def __call__(self, **arguments: Any) -> FunctionApplication:
|
||||
"""Bind this exact version to named table columns and constant inputs.
|
||||
|
||||
@@ -769,6 +871,9 @@ class FunctionRegistrationRequest(_RemoteValue):
|
||||
Credential values deliberately have no field here. The only secret-shaped
|
||||
thing a client sends is ``secret_bindings``: the name of a Secret the
|
||||
database already holds, which the remote service resolves at execution.
|
||||
``refresh_lock`` is a per-registration action, not part of the Function's
|
||||
source or runtime identity; its false default is omitted from canonical
|
||||
JSON.
|
||||
"""
|
||||
|
||||
name: str
|
||||
@@ -781,6 +886,7 @@ class FunctionRegistrationRequest(_RemoteValue):
|
||||
retry_policy: Optional[FunctionRetryPolicy] = None
|
||||
secret_bindings: tuple[SecretBinding, ...] = ()
|
||||
models: tuple[HuggingFaceModel, ...] = ()
|
||||
refresh_lock: bool = False
|
||||
|
||||
@field_validator("batch_size", mode="before")
|
||||
@classmethod
|
||||
@@ -797,6 +903,18 @@ class FunctionRegistrationRequest(_RemoteValue):
|
||||
def _validate_row_timeout(cls, value):
|
||||
return _validate_row_timeout(value)
|
||||
|
||||
@field_validator("refresh_lock", mode="before")
|
||||
@classmethod
|
||||
def _validate_refresh_lock(cls, value):
|
||||
return _validate_refresh_lock(value)
|
||||
|
||||
def with_refresh_lock(self, refresh_lock: bool) -> FunctionRegistrationRequest:
|
||||
"""Return this request with the registration-time refresh action set."""
|
||||
refresh_lock = _validate_refresh_lock(refresh_lock)
|
||||
if self.refresh_lock == refresh_lock:
|
||||
return self
|
||||
return self._copy(update={"refresh_lock": refresh_lock})
|
||||
|
||||
|
||||
class FunctionVersionRef(_OpenRemoteValue):
|
||||
name: str
|
||||
@@ -3043,6 +3161,7 @@ __all__ = [
|
||||
"FunctionArtifactContent",
|
||||
"FunctionArtifactRequest",
|
||||
"FunctionBinding",
|
||||
"FunctionEnvironment",
|
||||
"FunctionOutput",
|
||||
"FunctionParameter",
|
||||
"FunctionRegistrationRequest",
|
||||
@@ -3059,6 +3178,9 @@ __all__ = [
|
||||
"PythonEnvironmentSpec",
|
||||
"PythonAdapterSpec",
|
||||
"PythonRuntimeSpec",
|
||||
"ResolvedEnvironment",
|
||||
"ResolvedEnvironmentProvenance",
|
||||
"ResolvedPackage",
|
||||
"RefreshColumnResult",
|
||||
"FunctionDefinition",
|
||||
]
|
||||
|
||||
@@ -1002,6 +1002,7 @@ class RemoteDBConnection(DBConnection):
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
refresh_lock: bool = False,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(
|
||||
self._conn.create_function_async(
|
||||
@@ -1009,6 +1010,7 @@ class RemoteDBConnection(DBConnection):
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
refresh_lock=refresh_lock,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
|
||||
@@ -26,8 +26,9 @@ import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb import BlobReader
|
||||
from lancedb.db import AsyncConnection
|
||||
from lancedb.db import AsyncConnection, DBConnection, LanceDBConnection
|
||||
from lancedb.functions import (
|
||||
FunctionRegistrationRequest,
|
||||
FunctionSignature,
|
||||
FunctionVersion,
|
||||
HuggingFaceModel,
|
||||
@@ -82,6 +83,63 @@ def analyze_caption(caption: str) -> str:
|
||||
return caption.strip()
|
||||
|
||||
|
||||
def _function_version_with_resolved_environment():
|
||||
version = json.loads(
|
||||
(FIXTURES / "remote_function_version.canonical.json").read_text()
|
||||
)
|
||||
version["runtime"] = json.loads(
|
||||
normalize_score.registration_request.runtime.to_canonical_json()
|
||||
)
|
||||
version["resolved_environment"] = {
|
||||
"packages": [
|
||||
{
|
||||
"name": "torch",
|
||||
"version": "2.9.0+cu130",
|
||||
"source": "https://packages.example/simple",
|
||||
},
|
||||
{"name": "transformers", "version": "4.57.1"},
|
||||
],
|
||||
"provenance": {
|
||||
"configured_index_urls": ["https://packages.example/simple"],
|
||||
"configured_find_links": ["https://downloads.example/wheels"],
|
||||
"resolved_indexes": ["https://packages.example/simple"],
|
||||
"cuda_toolkit": "12.8",
|
||||
"torch_version": "torch==2.9.0+cu130",
|
||||
},
|
||||
"warnings": [
|
||||
"transformers resolved to transformers==4.57.1 but is not pinned "
|
||||
"exactly; add transformers==4.57.1 to pin it."
|
||||
],
|
||||
}
|
||||
return version
|
||||
|
||||
|
||||
def _function_version_with_resolved_environments():
|
||||
version = json.loads(
|
||||
(FIXTURES / "remote_function_version.canonical.json").read_text()
|
||||
)
|
||||
version["runtime"] = json.loads(
|
||||
normalize_score.registration_request.runtime.to_canonical_json()
|
||||
)
|
||||
version["resolved_environments"] = {
|
||||
"linux/amd64": {
|
||||
"packages": [
|
||||
{
|
||||
"name": "torch",
|
||||
"version": "2.9.0+cu130",
|
||||
"source": "https://packages.example/simple",
|
||||
}
|
||||
],
|
||||
"warnings": ["amd64 environment warning"],
|
||||
},
|
||||
"linux/arm64": {
|
||||
"packages": [],
|
||||
"warnings": ["arm64 environment warning"],
|
||||
},
|
||||
}
|
||||
return version
|
||||
|
||||
|
||||
def test_scalar_function_matches_shared_registration_golden_and_remains_callable():
|
||||
assert isinstance(normalize_score, FunctionDefinition)
|
||||
assert normalize_score(25.0) == 0.25
|
||||
@@ -98,6 +156,118 @@ def test_scalar_function_matches_shared_registration_golden_and_remains_callable
|
||||
}
|
||||
|
||||
|
||||
def test_refresh_lock_is_a_registration_option_not_part_of_identity():
|
||||
default = normalize_score.registration_request
|
||||
old_json = default.to_canonical_json()
|
||||
assert "refresh_lock" not in json.loads(old_json)
|
||||
|
||||
refreshed = default.with_refresh_lock(True)
|
||||
payload = json.loads(refreshed.to_canonical_json())
|
||||
assert payload["refresh_lock"] is True
|
||||
assert (
|
||||
FunctionRegistrationRequest.from_json(
|
||||
refreshed.to_canonical_json()
|
||||
).refresh_lock
|
||||
is True
|
||||
)
|
||||
assert refreshed.artifact.digest == default.artifact.digest
|
||||
assert refreshed.runtime.digest == default.runtime.digest
|
||||
assert default.with_refresh_lock(False).to_canonical_json() == old_json
|
||||
with pytest.raises(ValueError, match="refresh_lock must be a boolean"):
|
||||
default.with_refresh_lock(1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("exist_ok", [False, True])
|
||||
async def test_async_create_function_forwards_refresh_lock(exist_ok):
|
||||
calls = []
|
||||
|
||||
class Inner:
|
||||
async def create_function_async(
|
||||
self, request_json, namespace_path=None, *, exist_ok=False
|
||||
):
|
||||
calls.append((json.loads(request_json), namespace_path, exist_ok))
|
||||
return object()
|
||||
|
||||
connection = object.__new__(AsyncConnection)
|
||||
connection._inner = Inner()
|
||||
await connection.create_function_async(normalize_score)
|
||||
await connection.create_function_async(
|
||||
normalize_score, refresh_lock=True, exist_ok=exist_ok, namespace_path=["team"]
|
||||
)
|
||||
with pytest.raises(ValueError, match="refresh_lock must be a boolean"):
|
||||
await connection.create_function_async(normalize_score, refresh_lock=1)
|
||||
|
||||
assert len(calls) == 2
|
||||
assert "refresh_lock" not in calls[0][0]
|
||||
assert calls[0][1:] == (None, False)
|
||||
assert calls[1][0]["refresh_lock"] is True
|
||||
assert calls[1][1:] == (["team"], exist_ok)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exist_ok", [False, True])
|
||||
def test_sync_create_function_paths_forward_refresh_lock(exist_ok):
|
||||
calls = []
|
||||
|
||||
class AsyncInner:
|
||||
async def create_function_async(
|
||||
self,
|
||||
definition,
|
||||
*,
|
||||
secrets=None,
|
||||
exist_ok=False,
|
||||
namespace_path=None,
|
||||
refresh_lock=False,
|
||||
):
|
||||
calls.append(("wrapped", refresh_lock, namespace_path, exist_ok))
|
||||
return object()
|
||||
|
||||
for connection_type in (LanceDBConnection, RemoteDBConnection):
|
||||
connection = object.__new__(connection_type)
|
||||
connection._conn = AsyncInner()
|
||||
connection.create_function_async(
|
||||
normalize_score,
|
||||
refresh_lock=True,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=["team"],
|
||||
)
|
||||
|
||||
class BlockingConnection:
|
||||
def create_function_async(
|
||||
self,
|
||||
definition,
|
||||
*,
|
||||
secrets=None,
|
||||
exist_ok=False,
|
||||
namespace_path=None,
|
||||
refresh_lock=False,
|
||||
):
|
||||
calls.append(("blocking", refresh_lock, namespace_path, exist_ok))
|
||||
|
||||
class ImmediateJob:
|
||||
def wait(self):
|
||||
return "version"
|
||||
|
||||
return ImmediateJob()
|
||||
|
||||
blocking = BlockingConnection()
|
||||
assert (
|
||||
DBConnection.create_function(
|
||||
blocking,
|
||||
normalize_score,
|
||||
refresh_lock=True,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=["team"],
|
||||
)
|
||||
== "version"
|
||||
)
|
||||
assert calls == [
|
||||
("wrapped", True, ["team"], exist_ok),
|
||||
("wrapped", True, ["team"], exist_ok),
|
||||
("blocking", True, ["team"], exist_ok),
|
||||
]
|
||||
|
||||
|
||||
def test_function_declares_pinned_hugging_face_models():
|
||||
model = HuggingFaceModel(
|
||||
repo="sentence-transformers/all-MiniLM-L6-v2",
|
||||
@@ -2205,6 +2375,205 @@ class _RawFunctionCatalog:
|
||||
return None
|
||||
|
||||
|
||||
def test_function_version_reads_requested_and_resolved_environments():
|
||||
legacy = json.loads(
|
||||
(FIXTURES / "remote_function_version.canonical.json").read_text()
|
||||
)
|
||||
legacy_version = FunctionVersion.from_json(json.dumps(legacy))
|
||||
assert legacy_version.runtime is None
|
||||
assert legacy_version.resolved_environment is None
|
||||
assert legacy_version.environment.requested is None
|
||||
assert legacy_version.environment.resolved is None
|
||||
|
||||
payload = _function_version_with_resolved_environment()
|
||||
version = FunctionVersion.from_json(json.dumps(payload))
|
||||
assert version.environment.requested is version.runtime
|
||||
assert version.environment.resolved is version.resolved_environment
|
||||
assert version.runtime.kind == "python"
|
||||
resolved = version.resolved_environment
|
||||
assert resolved.packages[0].version == "2.9.0+cu130"
|
||||
assert resolved.packages[0].source == "https://packages.example/simple"
|
||||
assert resolved.packages[1].source is None
|
||||
assert resolved.provenance.torch_version == "torch==2.9.0+cu130"
|
||||
assert resolved.warnings == tuple(payload["resolved_environment"]["warnings"])
|
||||
|
||||
encoded = json.loads(version.to_canonical_json())
|
||||
assert encoded["runtime"]["kind"] == "python"
|
||||
assert encoded["resolved_environment"]["packages"][0]["version"] == "2.9.0+cu130"
|
||||
assert "source" not in encoded["resolved_environment"]["packages"][1]
|
||||
assert (
|
||||
encoded["resolved_environment"]["provenance"]["torch_version"]
|
||||
== "torch==2.9.0+cu130"
|
||||
)
|
||||
assert FunctionVersion.from_json(version.to_canonical_json()) == version
|
||||
|
||||
invalid = json.loads(json.dumps(payload))
|
||||
del invalid["resolved_environment"]["packages"]
|
||||
with pytest.raises(ValueError):
|
||||
FunctionVersion.from_json(json.dumps(invalid))
|
||||
|
||||
|
||||
def test_function_version_reads_resolved_environments_by_platform():
|
||||
legacy = json.loads(
|
||||
(FIXTURES / "remote_function_version.canonical.json").read_text()
|
||||
)
|
||||
legacy_version = FunctionVersion.from_json(json.dumps(legacy))
|
||||
assert legacy_version.resolved_environment is None
|
||||
assert legacy_version.resolved_environments == {}
|
||||
assert legacy_version.environment_report == {}
|
||||
|
||||
payload = _function_version_with_resolved_environments()
|
||||
version = FunctionVersion.from_json(json.dumps(payload))
|
||||
assert version.resolved_environment is None
|
||||
assert version.resolved_environments == version.environment_report
|
||||
assert set(version.environment_report) == {"linux/amd64", "linux/arm64"}
|
||||
assert (
|
||||
version.environment_report["linux/amd64"].packages[0].version == "2.9.0+cu130"
|
||||
)
|
||||
assert version.environment_report["linux/arm64"].packages == ()
|
||||
assert version.environment_report["linux/arm64"].warnings == (
|
||||
"arm64 environment warning",
|
||||
)
|
||||
|
||||
encoded = json.loads(version.to_canonical_json())
|
||||
assert "resolved_environment" not in encoded
|
||||
assert (
|
||||
encoded["resolved_environments"]["linux/amd64"]["packages"][0]["version"]
|
||||
== "2.9.0+cu130"
|
||||
)
|
||||
assert encoded["resolved_environments"]["linux/arm64"]["packages"] == []
|
||||
assert FunctionVersion.from_json(version.to_canonical_json()) == version
|
||||
|
||||
missing_platform = json.loads(json.dumps(payload))
|
||||
del missing_platform["resolved_environments"]["linux/arm64"]
|
||||
missing_version = FunctionVersion.from_json(json.dumps(missing_platform))
|
||||
assert "linux/arm64" not in missing_version.environment_report
|
||||
assert "linux/amd64" in missing_version.environment_report
|
||||
|
||||
empty_map = json.loads(json.dumps(payload))
|
||||
empty_map["resolved_environments"] = {}
|
||||
empty_version = FunctionVersion.from_json(json.dumps(empty_map))
|
||||
assert empty_version.resolved_environments == {}
|
||||
assert "resolved_environments" not in json.loads(empty_version.to_canonical_json())
|
||||
|
||||
invalid = json.loads(json.dumps(payload))
|
||||
del invalid["resolved_environments"]["linux/amd64"]["packages"]
|
||||
with pytest.raises(ValueError):
|
||||
FunctionVersion.from_json(json.dumps(invalid))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registration_job_emits_server_environment_warnings_once():
|
||||
payload = _function_version_with_resolved_environment()
|
||||
|
||||
class InnerJob:
|
||||
async def wait(self):
|
||||
return json.dumps(payload)
|
||||
|
||||
class Inner:
|
||||
async def create_function_async(
|
||||
self, request_json, namespace_path=None, *, exist_ok=False
|
||||
):
|
||||
assert "refresh_lock" not in json.loads(request_json)
|
||||
return InnerJob()
|
||||
|
||||
connection = object.__new__(AsyncConnection)
|
||||
connection._inner = Inner()
|
||||
job = await connection.create_function_async(normalize_score)
|
||||
with pytest.warns(UserWarning) as recorded:
|
||||
version = await job.wait()
|
||||
assert [str(w.message) for w in recorded] == list(
|
||||
payload["resolved_environment"]["warnings"]
|
||||
)
|
||||
assert version.environment.resolved.warnings == tuple(
|
||||
payload["resolved_environment"]["warnings"]
|
||||
)
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
assert await job.wait() is not None
|
||||
assert caught == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_registration_job_emits_per_platform_environment_warnings():
|
||||
payload = _function_version_with_resolved_environments()
|
||||
|
||||
class InnerJob:
|
||||
async def wait(self):
|
||||
return json.dumps(payload)
|
||||
|
||||
class Inner:
|
||||
async def create_function_async(
|
||||
self, request_json, namespace_path=None, *, exist_ok=False
|
||||
):
|
||||
return InnerJob()
|
||||
|
||||
connection = object.__new__(AsyncConnection)
|
||||
connection._inner = Inner()
|
||||
job = await connection.create_function_async(normalize_score)
|
||||
with pytest.warns(UserWarning) as recorded:
|
||||
version = await job.wait()
|
||||
assert [str(w.message) for w in recorded] == [
|
||||
"amd64 environment warning",
|
||||
"arm64 environment warning",
|
||||
]
|
||||
assert version.environment_report["linux/arm64"].warnings == (
|
||||
"arm64 environment warning",
|
||||
)
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
assert await job.wait() is not None
|
||||
assert caught == []
|
||||
|
||||
|
||||
def test_blocking_create_function_emits_server_environment_warnings():
|
||||
payload = _function_version_with_resolved_environment()
|
||||
|
||||
class InnerJob:
|
||||
async def wait(self):
|
||||
return json.dumps(payload)
|
||||
|
||||
class Inner:
|
||||
async def create_function_async(
|
||||
self, request_json, namespace_path=None, *, exist_ok=False
|
||||
):
|
||||
return InnerJob()
|
||||
|
||||
connection = object.__new__(RemoteDBConnection)
|
||||
connection._conn = AsyncConnection(Inner())
|
||||
with pytest.warns(UserWarning) as recorded:
|
||||
version = DBConnection.create_function(connection, normalize_score)
|
||||
assert [str(w.message) for w in recorded] == list(
|
||||
payload["resolved_environment"]["warnings"]
|
||||
)
|
||||
assert version.environment.resolved.packages[0].version == "2.9.0+cu130"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_catalog_readback_does_not_replay_registration_warnings():
|
||||
payload = _function_version_with_resolved_environment()
|
||||
connection = AsyncConnection(_RawFunctionCatalog(payload, encode=True))
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
version = await connection.get_function("embed")
|
||||
listed = [item async for item in connection.list_functions()]
|
||||
assert caught == []
|
||||
assert version.environment.resolved.packages[0].version == "2.9.0+cu130"
|
||||
assert listed[0].environment.resolved == version.resolved_environment
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_catalog_readback_does_not_replay_per_platform_warnings():
|
||||
payload = _function_version_with_resolved_environments()
|
||||
connection = AsyncConnection(_RawFunctionCatalog(payload, encode=True))
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
version = await connection.get_function("embed")
|
||||
listed = [item async for item in connection.list_functions()]
|
||||
assert caught == []
|
||||
assert version.environment_report == listed[0].environment_report
|
||||
|
||||
|
||||
def _assert_execution_control_encoding(version):
|
||||
encoded = json.loads(version.to_canonical_json())
|
||||
assert type(encoded["batch_size"]) is int
|
||||
|
||||
Reference in New Issue
Block a user