mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
fix(python): warn when Function dependencies cannot be verified (#4879)
Functions that import a module unavailable on the client currently package silently even when they declare no pip/conda dependencies. Emit an advisory warning for these unresolved imports so callers can declare the required package or confirm that the base image supplies it. Any pip/conda declaration suppresses this new advisory because distribution names need not match import names. The warning does not claim that the module is absent remotely and follows normal Python warning filters, including warnings-as-errors. Validation includes the original `import scipy.ndimage` with `pip=()` through the public decorator/packager path, plus declared-package and existing local-source controls. This does not establish remote image availability or deployment acceptance. Refs ENT-3448.
This commit is contained in:
@@ -1878,9 +1878,12 @@ class _SourcePackager:
|
||||
return f"{name} = {_literal_source(value)}"
|
||||
|
||||
def _require_importable(self, module_name: str) -> None:
|
||||
"""Warn about an import the Function's environment likely cannot
|
||||
satisfy: a module that lives in a local source tree, is not shipped
|
||||
with ``code=``, and names no declared package.
|
||||
"""Warn about an import the Function's environment may not be able to
|
||||
satisfy.
|
||||
|
||||
An unresolved module warns only when the Function declares no package
|
||||
dependencies. A local source module warns unless ``code=`` ships it or
|
||||
a declared requirement matches its top-level name.
|
||||
|
||||
A warning rather than an error: a distribution name need not match
|
||||
the module it installs, so a declared package can still provide the
|
||||
@@ -1896,8 +1899,19 @@ class _SourcePackager:
|
||||
):
|
||||
return
|
||||
locations = _module_locations_for_name(top)
|
||||
if not locations:
|
||||
if not self._requirements:
|
||||
self._warned_imports.add(top)
|
||||
warnings.warn(
|
||||
f"@function source imports {top!r}, which is unavailable in the "
|
||||
"client environment and has no declared pip/conda dependencies. "
|
||||
"Remote availability cannot be verified; declare the required "
|
||||
"package or confirm that the base image provides it.",
|
||||
UserWarning,
|
||||
)
|
||||
return
|
||||
roots = _installed_roots()
|
||||
if not locations or any(location.startswith(roots) for location in locations):
|
||||
if any(location.startswith(roots) for location in locations):
|
||||
return
|
||||
self._warned_imports.add(top)
|
||||
warnings.warn(
|
||||
@@ -2482,13 +2496,15 @@ def function(
|
||||
|
||||
A module imported by the packaged source -- including a direct ``import``
|
||||
inside a function or method body -- draws a warning at registration when
|
||||
it resolves to local source outside ``code`` and no declared requirement
|
||||
matches the imported module name. The client cannot reliably validate
|
||||
imports provided only by the base image: ``pip``/``conda`` distribution
|
||||
names need not match the modules they install, and the client does not
|
||||
know the contents of a built or custom image. Declare every third-party
|
||||
import in ``pip`` or ``conda`` even when it happens to be installed on the
|
||||
registration host.
|
||||
the client cannot resolve its top-level module and declares no package
|
||||
dependencies, or when it resolves to local source outside ``code`` and
|
||||
no declared requirement matches the imported module name. The warning
|
||||
does not assert that the remote environment lacks the module:
|
||||
``pip``/``conda`` distribution names need not match the modules they
|
||||
install, and the client cannot inspect the contents of a base or custom
|
||||
image. Warnings follow normal Python warning filters and may be promoted
|
||||
to errors. Declare every third-party import in ``pip`` or ``conda`` even
|
||||
when it happens to be installed on the registration host.
|
||||
Closures, lambdas, nested definitions, mutable module-level
|
||||
objects, code that reaches the module namespace another way
|
||||
(``globals()``/``eval``, ``sys.modules``, ``builtins``), and a
|
||||
|
||||
@@ -575,6 +575,94 @@ def test_local_module_imported_in_function_body_warns_at_registration(
|
||||
assert "import fx_body_local" in artifact(installed).decode()
|
||||
|
||||
|
||||
def test_missing_module_without_declared_package_warns_at_registration(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
package = "fx_missing_remote_package_ent3448"
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
|
||||
def define(name):
|
||||
(tmp_path / f"{name}.py").write_text(
|
||||
textwrap.dedent(
|
||||
f"""
|
||||
from lancedb import function
|
||||
|
||||
|
||||
@function(pip=())
|
||||
def {name}(value: int) -> int:
|
||||
import {package}.plugin
|
||||
|
||||
return value
|
||||
"""
|
||||
)
|
||||
)
|
||||
importlib.invalidate_caches()
|
||||
return getattr(importlib.import_module(name), name)
|
||||
|
||||
with pytest.warns(UserWarning) as caught:
|
||||
definition = define("fx_uses_missing_package")
|
||||
|
||||
message = str(caught[0].message)
|
||||
assert f"imports {package!r}" in message
|
||||
assert "unavailable in the client environment" in message
|
||||
assert "no declared pip/conda dependencies" in message
|
||||
assert "Remote availability cannot be verified" in message
|
||||
assert "confirm that the base image provides it" in message
|
||||
assert "cannot import" not in message
|
||||
assert f"import {package}.plugin" in artifact(definition).decode()
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", UserWarning)
|
||||
with pytest.raises(UserWarning, match="Remote availability cannot be verified"):
|
||||
define("fx_missing_package_as_error")
|
||||
|
||||
|
||||
def test_resolved_or_declared_module_imports_do_not_warn(tmp_path, helper_package):
|
||||
package = "fx_missing_remote_package_ent3448"
|
||||
helper_package("fx_declared_code_package", {"__init__.py": ""})
|
||||
|
||||
def define(name, body, options, imports=""):
|
||||
(tmp_path / f"{name}.py").write_text(
|
||||
textwrap.dedent(
|
||||
f"""
|
||||
{imports}
|
||||
from lancedb import function
|
||||
|
||||
|
||||
@function({options})
|
||||
def {name}(value: int) -> int:
|
||||
{body}
|
||||
return value
|
||||
"""
|
||||
)
|
||||
)
|
||||
importlib.invalidate_caches()
|
||||
return getattr(importlib.import_module(name), name)
|
||||
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
warnings.simplefilter("always")
|
||||
define("fx_imports_stdlib", "import os.path", "")
|
||||
# Distribution names need not match the imported top-level module.
|
||||
define(
|
||||
"fx_imports_declared_pip",
|
||||
f"import {package}.plugin",
|
||||
"pip=['Pillow>=10']",
|
||||
)
|
||||
define(
|
||||
"fx_imports_declared_conda",
|
||||
f"import {package}.plugin",
|
||||
"conda=['numpy']",
|
||||
)
|
||||
define(
|
||||
"fx_imports_shipped_code",
|
||||
"import fx_declared_code_package",
|
||||
"code=[fx_declared_code_package]",
|
||||
"import fx_declared_code_package",
|
||||
)
|
||||
|
||||
assert not caught
|
||||
|
||||
|
||||
MMLB_HELPERS = {
|
||||
"__init__.py": "",
|
||||
"rate.py": """
|
||||
|
||||
Reference in New Issue
Block a user