diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 4ce325b2f..36f47c8f0 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -1420,3 +1420,15 @@ name; omitting it allows matching reads of any revision. Prefixes must be concre paths without wildcards or traversal. This integration does not make Hugging Face a writable database location for `ATTACH DATABASE`. Dataset SQL sources select a matching integration automatically, with no integration argument in the query. + + +### Function package resources + +`@function(code=[my_package])` includes the package’s Python modules and data +files, including vendored subpackages. Resource paths and bytes are preserved, +so `importlib.resources` and package integrity checks work after deployment. +Changing a resource changes the Function artifact digest. This requires a server +that accepts `python_bundle.resources`. + +Caches, hidden paths, symlinks and unsafe paths are excluded. Use `pip=` or +`conda=` for platform-specific dependencies and `models=` for large model snapshots. diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index bebbe1169..04c146bb1 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -499,13 +499,20 @@ def caption(jpeg: bytes) -> str: Each module in `code` is captured and shipped the same way the function -itself is: as source, snapshotted at registration. Only `.py` source files are -included; package data such as JSON, YAML, prompt, or tokenizer files is not -copied. `@function` warns before registration when it skips such files from a -package in `code`, naming the affected paths. To use a package that includes -the required resources, install it with `pip=` or `conda=` and remove that -package from `code`; otherwise the Python-only copied package shadows the -installed distribution. +itself is: as source, snapshotted at registration. Packages also include regular +data files such as JSON, YAML, prompts, and tokenizer files, including resources +inside vendored subpackages. Resource paths and bytes are preserved for +`importlib.resources` and package integrity checks. Changing a resource changes +the Function artifact's identity. Caches, hidden paths, symlinks, and unsafe +paths are excluded. + +Package resources require matching SDK and server support. Upgrade query nodes +and Function image-building workers before registering resource bundles; older +servers reject them. Older SDKs ship only Python source and warn when excluding +package data. On deployments without resource-bundle support, install the +package with `pip=` or `conda=` and omit it from `code` so the copied package does +not shadow the installed distribution. Use environment dependencies for +platform-specific packages and `models=` for large model snapshots. ## Drop it {#drop} diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 92be658a8..f04d3d481 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -22,6 +22,7 @@ import symtable import json import linecache import math +import os import re import sys import textwrap @@ -29,6 +30,7 @@ import types import warnings from collections.abc import Mapping, Sequence from datetime import date, datetime +from pathlib import Path from typing import ( Annotated, Any, @@ -2104,18 +2106,12 @@ class _SourcePackager: self._require_importable(node.module) -def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]: - """The Python source files of the modules and packages in ``code=``, - keyed by their path on the Function's import path. - - Package resources are not part of the source artifact. Warn about files - and excluded directories that the walk leaves behind instead of silently - producing a Function that will fail later on the worker. - """ - import os - from pathlib import Path - +def _code_files( + modules: Sequence[Any], +) -> tuple[dict[str, str], dict[str, str], frozenset[str]]: + """Source and byte-preserving package resources on the Function import path.""" files: dict[str, str] = {} + resources: dict[str, str] = {} names: set[str] = set() for module in modules: if not isinstance(module, types.ModuleType): @@ -2149,35 +2145,50 @@ def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]] sources = [] skipped = set() for directory, subdirectories, filenames in os.walk(root): - # Only importable modules travel: data files, caches, and - # directories that are not identifiers cannot be imported. kept = [] for child in sorted(subdirectories): - if child == "__pycache__": + path = Path(directory) / child + if child == "__pycache__" or child.startswith("."): continue - if _SOURCE_PATH_COMPONENT.fullmatch(child): + if not path.is_symlink() and re.fullmatch( + r"[A-Za-z0-9_][A-Za-z0-9_.-]*", child + ): kept.append(child) else: - relative = (Path(directory) / child).relative_to(root) + relative = path.relative_to(root) skipped.add(f"{name}/{relative.as_posix()}/") subdirectories[:] = kept for filename in sorted(filenames): + path = Path(directory) / filename stem, suffix = os.path.splitext(filename) - if suffix in {".pyc", ".pyo"}: + if suffix in {".pyc", ".pyo"} or filename.startswith("."): continue - if suffix == ".py" and _SOURCE_PATH_COMPONENT.fullmatch(stem): - sources.append(Path(directory) / filename) + relative = path.relative_to(root) + key = f"{name}/{relative.as_posix()}" + if ( + path.is_symlink() + or not path.is_file() + or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", filename) + ): + skipped.add(key) + elif suffix == ".py": + if all( + _SOURCE_PATH_COMPONENT.fullmatch(part) + for part in relative.with_suffix("").parts + ): + sources.append(path) + else: + skipped.add(key) else: - relative = (Path(directory) / filename).relative_to(root) - skipped.add(f"{name}/{relative.as_posix()}") + resources[key] = base64.b64encode(path.read_bytes()).decode( + "ascii" + ) if skipped: - skipped_paths = ", ".join(sorted(skipped)) warnings.warn( - f"code= ships Python .py modules only; package {name!r} " - f"will not include: {skipped_paths}. Install a package that " - "includes the required resources with pip= or conda=, and " - f"remove {name!r} from code= so the copied package cannot " - "shadow the installed distribution.", + "code= cannot ship these package paths: " + f"{', '.join(sorted(skipped))}. " + "Use regular files with safe relative names and importable " + "Python module names.", UserWarning, ) for path in sources: @@ -2188,12 +2199,12 @@ def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]] if not file or not file.endswith(".py"): raise ValueError(f"code= ships Python source; module {name!r} has none") files[f"{name}.py"] = _source_text(Path(file)) - return files, frozenset(names) + return files, resources, frozenset(names) def _source_text(path) -> str: try: - return path.read_text(encoding="utf-8") + return path.read_bytes().decode("utf-8") except UnicodeDecodeError as error: raise ValueError(f"code= ships UTF-8 source; {path} is not UTF-8") from error @@ -2449,14 +2460,17 @@ class FunctionDefinition: UserWarning, stacklevel=3, ) - code_files, code_modules = _code_files(code) + code_files, code_resources, code_modules = _code_files(code) entry_source = _SourcePackager( code_modules, frozenset(_requirement_name(package) for package in packages) ).package(function, signature) if code_files: kind = _PYTHON_BUNDLE_ARTIFACT + bundle = {"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}} + if code_resources: + bundle["resources"] = code_resources source = json.dumps( - {"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}}, + bundle, ensure_ascii=False, sort_keys=True, separators=(",", ":"), @@ -2802,13 +2816,15 @@ def function( immutable Function version; per-refresh aggregate GPU budgets are selected through ``refresh_column`` instead of registration. code : sequence of module, optional - Top-level modules or packages whose importable ``.py`` files ship - with the Function and are importable by their own names. Package data - files are not copied. To use a package that includes required data - resources, install it with ``pip=`` or ``conda=`` and remove that - package from ``code=`` so the copied Python package cannot shadow the - installed distribution. ``@function`` warns when it skips files from a - package in ``code=``. + Top-level modules or packages shipped with the Function and importable + by their own names. Package resources, including binary data and + vendored subpackages, retain their relative paths and exact bytes. + Source and resource changes both create a new Function version. + Caches and hidden paths are excluded. Symlinks and unsafe paths are + skipped with a warning. Compiled dependencies must match the remote + runtime; use ``pip=`` or ``conda=`` for platform-specific distributions + and ``models=`` for large model snapshots. Resource bundles require + server support for the ``python_bundle.resources`` field. models : sequence of HuggingFaceModel, optional Immutable Hugging Face repository snapshots prepared in the worker node cache before user code starts. A Function may declare a diff --git a/python/python/tests/test_function_packaging.py b/python/python/tests/test_function_packaging.py index 2e285aa33..fcefc9110 100644 --- a/python/python/tests/test_function_packaging.py +++ b/python/python/tests/test_function_packaging.py @@ -7,6 +7,7 @@ from __future__ import annotations import base64 import importlib +import hashlib import json import subprocess import sys @@ -628,6 +629,102 @@ def test_binding_passes_initialization_as_constants(): version(text=col("body"), model=object()) +def test_vendored_package_resources_survive_an_isolated_import( + tmp_path, helper_package +): + package = helper_package( + "fx_vendor", + { + "__init__.py": "from .vendor.model import predict", + "vendor/__init__.py": "", + "vendor/model.py": """ + import hashlib + import json + from importlib.resources import files + + def predict(value): + root = files("fx_vendor") + payload = root.joinpath("label-maps/payload.bin").read_bytes() + metadata = json.loads(root.joinpath("models.json").read_text()) + source = root.joinpath("vendor/model.py").read_bytes() + source_digest = hashlib.sha256(source).hexdigest() + assert source_digest == metadata["source_sha256"] + assert hashlib.sha256(payload).hexdigest() == metadata["sha256"] + return value + metadata["offset"] + """, + "models.json": "{}", + }, + ) + payload = b"\x00\xff\r\nvendored model data" + package_root = Path(package.__file__).parent + (package_root / "label-maps").mkdir() + (package_root / "label-maps/payload.bin").write_bytes(payload) + module_path = package_root / "vendor/model.py" + module_bytes = ( + module_path.read_bytes().replace(b"\r\n", b"\n").replace(b"\n", b"\r\n") + ) + module_path.write_bytes(module_bytes) + (package_root / "models.json").write_bytes( + json.dumps( + { + "sha256": hashlib.sha256(payload).hexdigest(), + "source_sha256": hashlib.sha256(module_bytes).hexdigest(), + "offset": 7, + } + ).encode() + + b"\r\n" + ) + + entry = helper_package( + "fx_vendor_entry", + { + "__init__.py": """ + import fx_vendor + from lancedb import function + + @function(code=[fx_vendor]) + def vendored(value: int) -> int: + return fx_vendor.predict(value) + """ + }, + ) + vendored = entry.vendored + + first_digest = vendored.registration_request.artifact.digest + bundle = json.loads(artifact(vendored)) + isolated = tmp_path / "isolated" + isolated.mkdir() + for name, content in bundle["files"].items(): + target = isolated / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content.encode("utf-8")) + for name, content in bundle["resources"].items(): + target = isolated / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(base64.b64decode(content)) + assert (isolated / "fx_vendor/models.json").read_bytes().endswith(b"\r\n") + result = subprocess.run( + [ + sys.executable, + "-I", + "-c", + "import sys; sys.path.insert(0, sys.argv[1]); " + "from user_function import vendored; assert vendored(3) == 10", + str(isolated), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + (package_root / "models.json").write_text('{"offset": 8}') + assert ( + function(code=[package])( + vendored._function + ).registration_request.artifact.digest + != first_digest + ) + + def test_code_packages_ship_as_a_bundle_and_version_with_their_source( tmp_path, helper_package ): @@ -672,14 +769,13 @@ def test_code_packages_ship_as_a_bundle_and_version_with_their_source( first = define() assert len(caught) == 1 warning = str(caught[0].message) - assert "fx_helpers/fixtures/data.json" in warning + assert "fx_helpers/fixtures/data.json" not in warning assert "fx_helpers/not-a-module.py" in warning - assert warning.index("fx_helpers/fixtures/data.json") < warning.index( - "fx_helpers/not-a-module.py" - ) - assert "remove 'fx_helpers' from code=" in warning - assert "pip=" in warning - assert "conda=" in warning + + assert "safe relative names" in warning + assert "importable Python module names" in warning + resources = json.loads(artifact(first))["resources"] + assert base64.b64decode(resources["fx_helpers/fixtures/data.json"]) == b"{}" assert "__pycache__" not in warning assert ".pyc" not in warning shipped = bundle_files(first) @@ -732,7 +828,7 @@ def test_code_package_cache_artifacts_do_not_warn(helper_package): ] -def test_code_package_nonidentifier_resource_dir_warns_at_boundary(helper_package): +def test_code_package_resources_keep_nonidentifier_directories(helper_package): resources = helper_package( "fx_resources", { @@ -748,11 +844,9 @@ def test_code_package_nonidentifier_resource_dir_warns_at_boundary(helper_packag def resource_boundary(value: int) -> int: return value - assert len(caught) == 1 - warning = str(caught[0].message) - assert "fx_resources/label-maps/" in warning - assert "remove 'fx_resources' from code=" in warning - assert "labels.json" not in warning + assert not caught + resources = json.loads(artifact(resource_boundary))["resources"] + assert base64.b64decode(resources["fx_resources/label-maps/labels.json"]) == b"{}" assert sorted(bundle_files(resource_boundary)) == [ "fx_resources/__init__.py", "user_function.py",