feat(python): bundle Function package resources (#4987)

A package shipped through `code=` currently loses its non-Python files.
This breaks vendored packages that read JSON configuration, manifests,
prompts, or binary resources after deployment. Include regular package
resources in a separate base64-encoded `python_bundle.resources` map,
preserving exact bytes and relative paths. Both source and resources
contribute to the artifact digest. UTF-8 Python source bytes retain
their original line endings as well, so manifests that hash vendor code
continue to match.

For example, `@function(code=[my_package])` can use
`importlib.resources.files("my_package").joinpath("models.json").read_text()`
on the worker. Hidden files and caches are excluded; symlinks and unsafe
paths warn instead of following files outside the package. Source-only
bundles retain their existing wire form. Resource bundles require
matching server support.

Adds an isolated-import regression with a vendored subpackage, JSON
metadata, binary data, SHA256 verification, preserved CRLF bytes, and
resource-only version changes. Local builds and tests are disabled by
request. Ruff formatting and linting run locally; CI runs the tests.

docs/src/python/python.md

@@ -1420,3 +1420,15 @@ name; omitting it allows matching reads of any revision. Prefixes must be concre
 paths without wildcards or traversal. This integration does not make Hugging Face
 a writable database location for `ATTACH DATABASE`. Dataset SQL sources select a
 matching integration automatically, with no integration argument in the query.
+
+
+### Function package resources
+
+`@function(code=[my_package])` includes the package’s Python modules and data
+files, including vendored subpackages. Resource paths and bytes are preserved,
+so `importlib.resources` and package integrity checks work after deployment.
+Changing a resource changes the Function artifact digest. This requires a server
+that accepts `python_bundle.resources`.
+
+Caches, hidden paths, symlinks and unsafe paths are excluded. Use `pip=` or
+`conda=` for platform-specific dependencies and `models=` for large model snapshots.

docs/web/management/function.mdx

@@ -499,13 +499,20 @@ def caption(jpeg: bytes) -> str:
 </CodeGroup>
 
 Each module in `code` is captured and shipped the same way the function
-itself is: as source, snapshotted at registration. Only `.py` source files are
-included; package data such as JSON, YAML, prompt, or tokenizer files is not
-copied. `@function` warns before registration when it skips such files from a
-package in `code`, naming the affected paths. To use a package that includes
-the required resources, install it with `pip=` or `conda=` and remove that
-package from `code`; otherwise the Python-only copied package shadows the
-installed distribution.
+itself is: as source, snapshotted at registration. Packages also include regular
+data files such as JSON, YAML, prompts, and tokenizer files, including resources
+inside vendored subpackages. Resource paths and bytes are preserved for
+`importlib.resources` and package integrity checks. Changing a resource changes
+the Function artifact's identity. Caches, hidden paths, symlinks, and unsafe
+paths are excluded.
+
+Package resources require matching SDK and server support. Upgrade query nodes
+and Function image-building workers before registering resource bundles; older
+servers reject them. Older SDKs ship only Python source and warn when excluding
+package data. On deployments without resource-bundle support, install the
+package with `pip=` or `conda=` and omit it from `code` so the copied package does
+not shadow the installed distribution. Use environment dependencies for
+platform-specific packages and `models=` for large model snapshots.
 
 ## Drop it {#drop}
 

python/python/lancedb/functions.py

@@ -22,6 +22,7 @@ import symtable
 import json
 import linecache
 import math
+import os
 import re
 import sys
 import textwrap
@@ -29,6 +30,7 @@ import types
 import warnings
 from collections.abc import Mapping, Sequence
 from datetime import date, datetime
+from pathlib import Path
 from typing import (
     Annotated,
     Any,
@@ -2104,18 +2106,12 @@ class _SourcePackager:
                 self._require_importable(node.module)
 
 
-def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]:
-    """The Python source files of the modules and packages in ``code=``,
-    keyed by their path on the Function's import path.
-
-    Package resources are not part of the source artifact. Warn about files
-    and excluded directories that the walk leaves behind instead of silently
-    producing a Function that will fail later on the worker.
-    """
-    import os
-    from pathlib import Path
-
+def _code_files(
+    modules: Sequence[Any],
+) -> tuple[dict[str, str], dict[str, str], frozenset[str]]:
+    """Source and byte-preserving package resources on the Function import path."""
     files: dict[str, str] = {}
+    resources: dict[str, str] = {}
     names: set[str] = set()
     for module in modules:
         if not isinstance(module, types.ModuleType):
@@ -2149,35 +2145,50 @@ def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]
             sources = []
             skipped = set()
             for directory, subdirectories, filenames in os.walk(root):
-                # Only importable modules travel: data files, caches, and
-                # directories that are not identifiers cannot be imported.
                 kept = []
                 for child in sorted(subdirectories):
-                    if child == "__pycache__":
+                    path = Path(directory) / child
+                    if child == "__pycache__" or child.startswith("."):
                         continue
-                    if _SOURCE_PATH_COMPONENT.fullmatch(child):
+                    if not path.is_symlink() and re.fullmatch(
+                        r"[A-Za-z0-9_][A-Za-z0-9_.-]*", child
+                    ):
                         kept.append(child)
                     else:
-                        relative = (Path(directory) / child).relative_to(root)
+                        relative = path.relative_to(root)
                         skipped.add(f"{name}/{relative.as_posix()}/")
                 subdirectories[:] = kept
                 for filename in sorted(filenames):
+                    path = Path(directory) / filename
                     stem, suffix = os.path.splitext(filename)
-                    if suffix in {".pyc", ".pyo"}:
+                    if suffix in {".pyc", ".pyo"} or filename.startswith("."):
                         continue
-                    if suffix == ".py" and _SOURCE_PATH_COMPONENT.fullmatch(stem):
-                        sources.append(Path(directory) / filename)
+                    relative = path.relative_to(root)
+                    key = f"{name}/{relative.as_posix()}"
+                    if (
+                        path.is_symlink()
+                        or not path.is_file()
+                        or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", filename)
+                    ):
+                        skipped.add(key)
+                    elif suffix == ".py":
+                        if all(
+                            _SOURCE_PATH_COMPONENT.fullmatch(part)
+                            for part in relative.with_suffix("").parts
+                        ):
+                            sources.append(path)
+                        else:
+                            skipped.add(key)
                     else:
-                        relative = (Path(directory) / filename).relative_to(root)
-                        skipped.add(f"{name}/{relative.as_posix()}")
+                        resources[key] = base64.b64encode(path.read_bytes()).decode(
+                            "ascii"
+                        )
             if skipped:
-                skipped_paths = ", ".join(sorted(skipped))
                 warnings.warn(
-                    f"code= ships Python .py modules only; package {name!r} "
-                    f"will not include: {skipped_paths}. Install a package that "
-                    "includes the required resources with pip= or conda=, and "
-                    f"remove {name!r} from code= so the copied package cannot "
-                    "shadow the installed distribution.",
+                    "code= cannot ship these package paths: "
+                    f"{', '.join(sorted(skipped))}. "
+                    "Use regular files with safe relative names and importable "
+                    "Python module names.",
                     UserWarning,
                 )
             for path in sources:
@@ -2188,12 +2199,12 @@ def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]
             if not file or not file.endswith(".py"):
                 raise ValueError(f"code= ships Python source; module {name!r} has none")
             files[f"{name}.py"] = _source_text(Path(file))
-    return files, frozenset(names)
+    return files, resources, frozenset(names)
 
 
 def _source_text(path) -> str:
     try:
-        return path.read_text(encoding="utf-8")
+        return path.read_bytes().decode("utf-8")
     except UnicodeDecodeError as error:
         raise ValueError(f"code= ships UTF-8 source; {path} is not UTF-8") from error
 
@@ -2449,14 +2460,17 @@ class FunctionDefinition:
                     UserWarning,
                     stacklevel=3,
                 )
-        code_files, code_modules = _code_files(code)
+        code_files, code_resources, code_modules = _code_files(code)
         entry_source = _SourcePackager(
             code_modules, frozenset(_requirement_name(package) for package in packages)
         ).package(function, signature)
         if code_files:
             kind = _PYTHON_BUNDLE_ARTIFACT
+            bundle = {"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}}
+            if code_resources:
+                bundle["resources"] = code_resources
             source = json.dumps(
-                {"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}},
+                bundle,
                 ensure_ascii=False,
                 sort_keys=True,
                 separators=(",", ":"),
@@ -2802,13 +2816,15 @@ def function(
         immutable Function version; per-refresh aggregate GPU budgets are
         selected through ``refresh_column`` instead of registration.
     code : sequence of module, optional
-        Top-level modules or packages whose importable ``.py`` files ship
-        with the Function and are importable by their own names. Package data
-        files are not copied. To use a package that includes required data
-        resources, install it with ``pip=`` or ``conda=`` and remove that
-        package from ``code=`` so the copied Python package cannot shadow the
-        installed distribution. ``@function`` warns when it skips files from a
-        package in ``code=``.
+        Top-level modules or packages shipped with the Function and importable
+        by their own names. Package resources, including binary data and
+        vendored subpackages, retain their relative paths and exact bytes.
+        Source and resource changes both create a new Function version.
+        Caches and hidden paths are excluded. Symlinks and unsafe paths are
+        skipped with a warning. Compiled dependencies must match the remote
+        runtime; use ``pip=`` or ``conda=`` for platform-specific distributions
+        and ``models=`` for large model snapshots. Resource bundles require
+        server support for the ``python_bundle.resources`` field.
     models : sequence of HuggingFaceModel, optional
         Immutable Hugging Face repository snapshots prepared in the worker
         node cache before user code starts. A Function may declare a

python/python/tests/test_function_packaging.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import base64
 import importlib
+import hashlib
 import json
 import subprocess
 import sys
@@ -628,6 +629,102 @@ def test_binding_passes_initialization_as_constants():
         version(text=col("body"), model=object())
 
 
+def test_vendored_package_resources_survive_an_isolated_import(
+    tmp_path, helper_package
+):
+    package = helper_package(
+        "fx_vendor",
+        {
+            "__init__.py": "from .vendor.model import predict",
+            "vendor/__init__.py": "",
+            "vendor/model.py": """
+                import hashlib
+                import json
+                from importlib.resources import files
+
+                def predict(value):
+                    root = files("fx_vendor")
+                    payload = root.joinpath("label-maps/payload.bin").read_bytes()
+                    metadata = json.loads(root.joinpath("models.json").read_text())
+                    source = root.joinpath("vendor/model.py").read_bytes()
+                    source_digest = hashlib.sha256(source).hexdigest()
+                    assert source_digest == metadata["source_sha256"]
+                    assert hashlib.sha256(payload).hexdigest() == metadata["sha256"]
+                    return value + metadata["offset"]
+            """,
+            "models.json": "{}",
+        },
+    )
+    payload = b"\x00\xff\r\nvendored model data"
+    package_root = Path(package.__file__).parent
+    (package_root / "label-maps").mkdir()
+    (package_root / "label-maps/payload.bin").write_bytes(payload)
+    module_path = package_root / "vendor/model.py"
+    module_bytes = (
+        module_path.read_bytes().replace(b"\r\n", b"\n").replace(b"\n", b"\r\n")
+    )
+    module_path.write_bytes(module_bytes)
+    (package_root / "models.json").write_bytes(
+        json.dumps(
+            {
+                "sha256": hashlib.sha256(payload).hexdigest(),
+                "source_sha256": hashlib.sha256(module_bytes).hexdigest(),
+                "offset": 7,
+            }
+        ).encode()
+        + b"\r\n"
+    )
+
+    entry = helper_package(
+        "fx_vendor_entry",
+        {
+            "__init__.py": """
+        import fx_vendor
+        from lancedb import function
+
+        @function(code=[fx_vendor])
+        def vendored(value: int) -> int:
+            return fx_vendor.predict(value)
+    """
+        },
+    )
+    vendored = entry.vendored
+
+    first_digest = vendored.registration_request.artifact.digest
+    bundle = json.loads(artifact(vendored))
+    isolated = tmp_path / "isolated"
+    isolated.mkdir()
+    for name, content in bundle["files"].items():
+        target = isolated / name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(content.encode("utf-8"))
+    for name, content in bundle["resources"].items():
+        target = isolated / name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_bytes(base64.b64decode(content))
+    assert (isolated / "fx_vendor/models.json").read_bytes().endswith(b"\r\n")
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-I",
+            "-c",
+            "import sys; sys.path.insert(0, sys.argv[1]); "
+            "from user_function import vendored; assert vendored(3) == 10",
+            str(isolated),
+        ],
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stderr
+    (package_root / "models.json").write_text('{"offset": 8}')
+    assert (
+        function(code=[package])(
+            vendored._function
+        ).registration_request.artifact.digest
+        != first_digest
+    )
+
+
 def test_code_packages_ship_as_a_bundle_and_version_with_their_source(
     tmp_path, helper_package
 ):
@@ -672,14 +769,13 @@ def test_code_packages_ship_as_a_bundle_and_version_with_their_source(
         first = define()
     assert len(caught) == 1
     warning = str(caught[0].message)
-    assert "fx_helpers/fixtures/data.json" in warning
+    assert "fx_helpers/fixtures/data.json" not in warning
     assert "fx_helpers/not-a-module.py" in warning
-    assert warning.index("fx_helpers/fixtures/data.json") < warning.index(
-        "fx_helpers/not-a-module.py"
-    )
-    assert "remove 'fx_helpers' from code=" in warning
-    assert "pip=" in warning
-    assert "conda=" in warning
+
+    assert "safe relative names" in warning
+    assert "importable Python module names" in warning
+    resources = json.loads(artifact(first))["resources"]
+    assert base64.b64decode(resources["fx_helpers/fixtures/data.json"]) == b"{}"
     assert "__pycache__" not in warning
     assert ".pyc" not in warning
     shipped = bundle_files(first)
@@ -732,7 +828,7 @@ def test_code_package_cache_artifacts_do_not_warn(helper_package):
     ]
 
 
-def test_code_package_nonidentifier_resource_dir_warns_at_boundary(helper_package):
+def test_code_package_resources_keep_nonidentifier_directories(helper_package):
     resources = helper_package(
         "fx_resources",
         {
@@ -748,11 +844,9 @@ def test_code_package_nonidentifier_resource_dir_warns_at_boundary(helper_packag
         def resource_boundary(value: int) -> int:
             return value
 
-    assert len(caught) == 1
-    warning = str(caught[0].message)
-    assert "fx_resources/label-maps/" in warning
-    assert "remove 'fx_resources' from code=" in warning
-    assert "labels.json" not in warning
+    assert not caught
+    resources = json.loads(artifact(resource_boundary))["resources"]
+    assert base64.b64decode(resources["fx_resources/label-maps/labels.json"]) == b"{}"
     assert sorted(bundle_files(resource_boundary)) == [
         "fx_resources/__init__.py",
         "user_function.py",