From e61e21035c1f6f9b357510df5e88d0fa4c1be3f8 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 12 Nov 2026 09:01:39 -0800 Subject: [PATCH] fix(python): explain Function namespace import restrictions (#4982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A plain `import ctypes.util` in a Function currently reports “dynamic namespace access” without explaining the module restriction or a way forward. Give blocked module imports a distinct error that names the modules, explains the reconstructed namespace constraint, and recommends an importable helper supplied through `code=`, `pip=`, or `conda=`. Dynamic namespace operations also get an actionable explanation. Document the full restrictions and the helper alternative in the decorator and Python Functions reference. Add regression coverage for direct imports, `from` imports, global module aliases, class Functions, the deprecated `@udf` alias, and helpers imported normally or shipped with `code=`. Existing import restrictions remain in place. For example, move `ctypes.util.find_library` into `native_diagnostics/__init__.py`, then use: ```python import native_diagnostics from lancedb import function @function(code=[native_diagnostics]) def probe_library(name: str) -> str: return native_diagnostics.find_library(name) ``` Fixes ENT-3408. --- docs/src/python/python.md | 45 +++++++ python/python/lancedb/functions.py | 63 ++++++--- .../tests/test_first_class_function_slice2.py | 7 +- .../python/tests/test_function_packaging.py | 120 ++++++++++++++++++ 4 files changed, 214 insertions(+), 21 deletions(-) diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 8de5b5517..35f50c5a8 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -447,6 +447,51 @@ assert scaler(3.0) == 6.0 scaler.close() ``` +### Function packaging restrictions + +The decorator snapshots the Function's source and referenced globals into a new +module. It conservatively rejects imports of `sys`, `builtins`, `importlib`, +`inspect`, `gc`, `ctypes`, and `types`, including submodules such as `ctypes.util` +and `importlib.metadata`. This applies to imports inside function or class bodies +and references to module-level aliases. Even ordinary uses such as `sys.version` +and `ctypes.util.find_library` are blocked: these modules can reach names outside +the reconstructed Function namespace. References to `globals`, `locals`, `vars`, +`eval`, `exec`, `compile`, and `__import__` are rejected as dynamic namespace access. +These restrictions also apply to the deprecated `@udf` alias. + +For legitimate uses, move the code into an importable helper module and ship it +with `code=`, or install its package with `pip=` or `conda=`. Imported helpers keep +their own namespace; the packager does not scan their bodies. Helpers defined in +`__main__` are copied by source and have the same restrictions as the Function. + +For example, put this helper in `native_diagnostics/__init__.py`: + +```python +import ctypes.util + + +def find_library(name: str) -> str: + return ctypes.util.find_library(name) or "" +``` + +Then import the helper and include its source in the Function's artifact: + +```python +import native_diagnostics +from lancedb import function + + +@function(code=[native_diagnostics]) +def probe_library(name: str) -> str: + return native_diagnostics.find_library(name) +``` + +Captured literal globals support `None`, booleans, integers, strings, bytes, +finite floats, and tuples of these values. Construct other objects, including +PyArrow types and schemas, inside the Function or helper module instead. Values +used only in the decorator's `input_schema` or `output_schema` are not captured +globals and can be defined outside the Function. + Annotate a blob input with `lancedb.BlobReader` for lazy, seekable access, or with `bytes` to materialize its contents before the Function runs. Reader inputs infer the Blob V2 type without an explicit input schema. Readers are diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index a5c741cd8..19d4171fb 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -1570,10 +1570,23 @@ _NAMESPACE_MODULES = frozenset( ) -def _namespace_acquisition( - definition: ast.FunctionDef, references: set[str] -) -> list[str]: - found = set(references & _DYNAMIC_NAMESPACE_ACCESS) +_NAMESPACE_HELPER_HINT = ( + "Move this code into an importable helper module supplied with code=, pip=, " + "or conda=, then import and call the helper from the Function." +) + + +def _namespace_import_error(modules: Sequence[str]) -> ValueError: + return ValueError( + f"@function cannot package imports from namespace-access modules: {modules!r}. " + f"Imports of {', '.join(sorted(_NAMESPACE_MODULES))} (including submodules) " + "are blocked because they can access names outside the packaged " + "Function's reconstructed module namespace. " + _NAMESPACE_HELPER_HINT + ) + + +def _namespace_imports(definition: Union[ast.FunctionDef, ast.ClassDef]) -> list[str]: + found: set[str] = set() for node in ast.walk(definition): if isinstance(node, ast.Import): found.update( @@ -1883,11 +1896,17 @@ class _SourcePackager: ) source = ast.unparse(definition) references = _module_references(f"{_SOURCE_HEADER}\n\n{source}\n") - dynamic = _namespace_acquisition(definition, references) + dynamic = sorted(references & _DYNAMIC_NAMESPACE_ACCESS) if dynamic: raise ValueError( - f"@function cannot package dynamic namespace access: {dynamic!r}" + f"@function cannot package dynamic namespace access: {dynamic!r}. " + "These operations can access names outside the packaged " + "Function's reconstructed module namespace. Reference dependencies " + "explicitly. " + _NAMESPACE_HELPER_HINT ) + namespace_imports = _namespace_imports(definition) + if namespace_imports: + raise _namespace_import_error(namespace_imports) self._warn_unshipped_direct_imports(definition) self._packaged.add(id(target)) self._claim(target.__name__, ("definition", id(target))) @@ -1982,10 +2001,7 @@ class _SourcePackager: return f"from lance_blob import BlobReader as {name}" if isinstance(value, types.ModuleType): if value.__name__.split(".")[0] in _NAMESPACE_MODULES: - raise ValueError( - "@function cannot package dynamic namespace access: " - f"{value.__name__!r}" - ) + raise _namespace_import_error([value.__name__]) try: imported = importlib.import_module(value.__name__) except ImportError: @@ -2663,13 +2679,28 @@ def function( Other global values, including Arrow types and schemas, lists, dictionaries, sets, and NumPy arrays, cannot be captured as literals. Define them inside the function or class body instead. - Closures, lambdas, nested definitions, mutable module-level - objects, code that reaches the module namespace another way - (``globals()``/``eval``, ``sys.modules``, ``builtins``), and a + Closures, lambdas, nested definitions, mutable module-level objects, and a non-standard ``__builtins__`` are rejected where they can be seen and - otherwise unsupported. Changing any packaged source, including a module in - ``code``, produces a new Function version; the environment built from - ``pip`` or ``conda`` is reused. + otherwise unsupported. + + **Namespace restrictions.** The packaged source cannot import ``sys``, + ``builtins``, ``importlib``, ``inspect``, ``gc``, ``ctypes``, or ``types``, + including submodules such as ``ctypes.util`` and ``importlib.metadata``. + Referencing a module-level alias for one of these modules is also rejected. + References to ``globals``, ``locals``, ``vars``, ``eval``, ``exec``, + ``compile``, or ``__import__`` are rejected as dynamic namespace access. + These restrictions are conservative: even ordinary uses such as + ``ctypes.util.find_library`` or ``sys.version`` are blocked because these + modules can reach names outside the reconstructed Function namespace. + Move such code into an importable helper module supplied with ``code=``, + ``pip=``, or ``conda=``, then import and call the helper from the Function. + Helpers imported normally keep their own module namespace; these checks do + not scan their bodies. A helper defined in ``__main__`` is packaged by source + and is subject to the same checks. + + Changing any packaged source, including a module in ``code``, produces a + new Function version; the environment built from ``pip`` or ``conda`` is + reused. Parameters ---------- diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index d99a7aa21..97c0a5f3c 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -888,7 +888,7 @@ def test_function_rejects_a_module_level_namespace_alias(tmp_path): spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - with pytest.raises(ValueError, match="dynamic namespace access"): + with pytest.raises(ValueError, match="imports from namespace-access modules"): function(module.score) @@ -899,7 +899,6 @@ def test_function_rejects_a_module_level_namespace_alias(tmp_path): "eval('THRESHOLD')", "(lambda g: g()['THRESHOLD'])(globals)", "__import__('sys').modules[__name__].THRESHOLD", - "sys.modules[__name__].THRESHOLD", ], ) def test_function_rejects_dynamic_namespace_access(access): @@ -911,9 +910,7 @@ def test_function_rejects_dynamic_namespace_access(access): ) with pytest.raises(ValueError, match="dynamic namespace access"): _package_from_text( - "def score(value: int) -> int:\n" - " import sys\n" - f" return value + {access}\n" + f"def score(value: int) -> int:\n return value + {access}\n" ) diff --git a/python/python/tests/test_function_packaging.py b/python/python/tests/test_function_packaging.py index 440adc4da..2e285aa33 100644 --- a/python/python/tests/test_function_packaging.py +++ b/python/python/tests/test_function_packaging.py @@ -466,6 +466,126 @@ def test_nested_class_and_closure_are_rejected_with_the_reason(): return value * factor +@pytest.mark.parametrize( + "module_name", + [ + "sys", + "builtins", + "importlib", + "inspect", + "gc", + "ctypes", + "types", + "ctypes.util", + "importlib.metadata", + ], +) +@pytest.mark.parametrize("import_style", ["import", "from", "global"]) +@pytest.mark.parametrize("class_function", [False, True]) +def test_namespace_module_errors_explain_the_rule_and_remedy( + helper_package, module_name, import_style, class_function +): + statement = f"import {module_name} as namespace_module" + body = f"{statement}\nreturn value\n" + preamble = "" + if import_style == "from": + body = f"from {module_name} import __name__\nreturn value\n" + elif import_style == "global": + preamble = statement + "\n" + body = "return str(namespace_module)\n" + if class_function: + definition = "class blocked:\n def __call__(self, value: str) -> str:\n" + source = preamble + definition + textwrap.indent(body, " ") + else: + definition = "def blocked(value: str) -> str:\n" + source = preamble + definition + textwrap.indent(body, " ") + module = helper_package("fx_namespace_import", {"__init__.py": source}) + + with pytest.raises(ValueError) as error: + function(module.blocked) + message = str(error.value) + assert "imports from namespace-access modules" in message + assert repr(module_name) in message + assert "builtins, ctypes, gc, importlib, inspect, sys, types" in message + assert "including submodules" in message + assert "reconstructed module namespace" in message + assert "importable helper module" in message + assert "code=, pip=, or conda=" in message + assert "dynamic namespace access" not in message + + +@pytest.mark.filterwarnings("ignore:lancedb.udf is deprecated:DeprecationWarning") +def test_udf_namespace_import_error_explains_the_helper_alternative(): + with pytest.raises(ValueError, match="ctypes.util.*importable helper module"): + + @udf + def probe(value: str) -> str: + import ctypes.util + + return str(ctypes.util.find_library(value)) + + +@pytest.mark.parametrize( + "operation", ["globals", "locals", "vars", "eval", "exec", "compile", "__import__"] +) +def test_dynamic_namespace_errors_name_the_operation_and_remedy( + helper_package, operation +): + module = helper_package( + "fx_namespace_operation", + { + "__init__.py": ( + f"def blocked(value: str) -> str:\n return str({operation})\n" + ) + }, + ) + with pytest.raises(ValueError) as error: + function(module.blocked) + message = str(error.value) + assert f"dynamic namespace access: ['{operation}']" in message + assert "Reference dependencies explicitly" in message + assert "importable helper module" in message + + +@pytest.mark.parametrize("ship_source", [False, True]) +def test_namespace_access_in_importable_helpers_keeps_its_module_namespace( + helper_package, ship_source +): + helpers = helper_package( + "fx_namespace_helper", + { + "__init__.py": """ + import ctypes.util + + def module_name(): + return ctypes.util.__name__ + """, + }, + ) + entry = helper_package( + "fx_namespace_entry", + { + "__init__.py": """ + from fx_namespace_helper import module_name + + def probe(value: str) -> str: + return module_name() + value + """, + }, + ) + options = {"code": [helpers]} if ship_source else {"pip": ["fx-namespace-helper"]} + definition = function(**options)(entry.probe) + if ship_source: + files = bundle_files(definition) + assert "import ctypes.util" in files["fx_namespace_helper/__init__.py"] + source = files["user_function.py"] + else: + source = artifact(definition).decode() + namespace = {} + exec(compile(source, "user_function.py", "exec"), namespace) + assert namespace["probe"]("!") == "ctypes.util!" + + def test_binding_passes_initialization_as_constants(): version = FunctionVersion.from_json( json.dumps(