rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(python): bundle Function package resources (#4987)

Browse Source 
A package shipped through `code=` currently loses its non-Python files.
This breaks vendored packages that read JSON configuration, manifests,
prompts, or binary resources after deployment. Include regular package
resources in a separate base64-encoded `python_bundle.resources` map,
preserving exact bytes and relative paths. Both source and resources
contribute to the artifact digest. UTF-8 Python source bytes retain
their original line endings as well, so manifests that hash vendor code
continue to match.

For example, `@function(code=[my_package])` can use
`importlib.resources.files("my_package").joinpath("models.json").read_text()`
on the worker. Hidden files and caches are excluded; symlinks and unsafe
paths warn instead of following files outside the package. Source-only
bundles retain their existing wire form. Resource bundles require
matching server support.

Adds an isolated-import regression with a vendored subpackage, JSON
metadata, binary data, SHA256 verification, preserved CRLF bytes, and
resource-only version changes. Local builds and tests are disabled by
request. Ruff formatting and linting run locally; CI runs the tests.
This commit is contained in:
Jack Ye
2026-11-13 16:21:58 +08:00
committed by GitHub
parent db098ba812
commit bab22a3fbd
4 changed files with 187 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/functions.py
+54 -38
View File
@@ -22,6 +22,7 @@ import symtable
import json
import linecache
import math
import os
import re
import sys
import textwrap
@@ -29,6 +30,7 @@ import types
import warnings
from collections.abc import Mapping, Sequence
from datetime import date, datetime
from pathlib import Path
from typing import (
Annotated,
Any,
@@ -2104,18 +2106,12 @@ class _SourcePackager:
self._require_importable(node.module)
def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]:
"""The Python source files of the modules and packages in ``code=``,
keyed by their path on the Function's import path.
Package resources are not part of the source artifact. Warn about files
and excluded directories that the walk leaves behind instead of silently
producing a Function that will fail later on the worker.
"""
import os
from pathlib import Path
def _code_files(
modules: Sequence[Any],
) -> tuple[dict[str, str], dict[str, str], frozenset[str]]:
"""Source and byte-preserving package resources on the Function import path."""
files: dict[str, str] = {}
resources: dict[str, str] = {}
names: set[str] = set()
for module in modules:
if not isinstance(module, types.ModuleType):
@@ -2149,35 +2145,50 @@ def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]
sources = []
skipped = set()
for directory, subdirectories, filenames in os.walk(root):
# Only importable modules travel: data files, caches, and
# directories that are not identifiers cannot be imported.
kept = []
for child in sorted(subdirectories):
if child == "__pycache__":
path = Path(directory) / child
if child == "__pycache__" or child.startswith("."):
continue
if _SOURCE_PATH_COMPONENT.fullmatch(child):
if not path.is_symlink() and re.fullmatch(
r"[A-Za-z0-9_][A-Za-z0-9_.-]*", child
):
kept.append(child)
else:
relative = (Path(directory) / child).relative_to(root)
relative = path.relative_to(root)
skipped.add(f"{name}/{relative.as_posix()}/")
subdirectories[:] = kept
for filename in sorted(filenames):
path = Path(directory) / filename
stem, suffix = os.path.splitext(filename)
if suffix in {".pyc", ".pyo"}:
if suffix in {".pyc", ".pyo"} or filename.startswith("."):
continue
if suffix == ".py" and _SOURCE_PATH_COMPONENT.fullmatch(stem):
sources.append(Path(directory) / filename)
relative = path.relative_to(root)
key = f"{name}/{relative.as_posix()}"
if (
path.is_symlink()
or not path.is_file()
or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", filename)
):
skipped.add(key)
elif suffix == ".py":
if all(
_SOURCE_PATH_COMPONENT.fullmatch(part)
for part in relative.with_suffix("").parts
):
sources.append(path)
else:
skipped.add(key)
else:
relative = (Path(directory) / filename).relative_to(root)
skipped.add(f"{name}/{relative.as_posix()}")
resources[key] = base64.b64encode(path.read_bytes()).decode(
"ascii"
)
if skipped:
skipped_paths = ", ".join(sorted(skipped))
warnings.warn(
f"code= ships Python .py modules only; package {name!r} "
f"will not include: {skipped_paths}. Install a package that "
"includes the required resources with pip= or conda=, and "
f"remove {name!r} from code= so the copied package cannot "
"shadow the installed distribution.",
"code= cannot ship these package paths: "
f"{', '.join(sorted(skipped))}. "
"Use regular files with safe relative names and importable "
"Python module names.",
UserWarning,
)
for path in sources:
@@ -2188,12 +2199,12 @@ def _code_files(modules: Sequence[Any]) -> tuple[dict[str, str], frozenset[str]]
if not file or not file.endswith(".py"):
raise ValueError(f"code= ships Python source; module {name!r} has none")
files[f"{name}.py"] = _source_text(Path(file))
return files, frozenset(names)
return files, resources, frozenset(names)
def _source_text(path) -> str:
try:
return path.read_text(encoding="utf-8")
return path.read_bytes().decode("utf-8")
except UnicodeDecodeError as error:
raise ValueError(f"code= ships UTF-8 source; {path} is not UTF-8") from error
@@ -2449,14 +2460,17 @@ class FunctionDefinition:
UserWarning,
stacklevel=3,
)
code_files, code_modules = _code_files(code)
code_files, code_resources, code_modules = _code_files(code)
entry_source = _SourcePackager(
code_modules, frozenset(_requirement_name(package) for package in packages)
).package(function, signature)
if code_files:
kind = _PYTHON_BUNDLE_ARTIFACT
bundle = {"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}}
if code_resources:
bundle["resources"] = code_resources
source = json.dumps(
{"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}},
bundle,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
@@ -2802,13 +2816,15 @@ def function(
immutable Function version; per-refresh aggregate GPU budgets are
selected through ``refresh_column`` instead of registration.
code : sequence of module, optional
Top-level modules or packages whose importable ``.py`` files ship
with the Function and are importable by their own names. Package data
files are not copied. To use a package that includes required data
resources, install it with ``pip=`` or ``conda=`` and remove that
package from ``code=`` so the copied Python package cannot shadow the
installed distribution. ``@function`` warns when it skips files from a
package in ``code=``.
Top-level modules or packages shipped with the Function and importable
by their own names. Package resources, including binary data and
vendored subpackages, retain their relative paths and exact bytes.
Source and resource changes both create a new Function version.
Caches and hidden paths are excluded. Symlinks and unsafe paths are
skipped with a warning. Compiled dependencies must match the remote
runtime; use ``pip=`` or ``conda=`` for platform-specific distributions
and ``models=`` for large model snapshots. Resource bundles require
server support for the ``python_bundle.resources`` field.
models : sequence of HuggingFaceModel, optional
Immutable Hugging Face repository snapshots prepared in the worker
node cache before user code starts. A Function may declare a
python/python/tests/test_function_packaging.py
+107 -13
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import base64
import importlib
import hashlib
import json
import subprocess
import sys
@@ -628,6 +629,102 @@ def test_binding_passes_initialization_as_constants():
version(text=col("body"), model=object())
def test_vendored_package_resources_survive_an_isolated_import(
tmp_path, helper_package
):
package = helper_package(
"fx_vendor",
{
"__init__.py": "from .vendor.model import predict",
"vendor/__init__.py": "",
"vendor/model.py": """
import hashlib
import json
from importlib.resources import files
def predict(value):
root = files("fx_vendor")
payload = root.joinpath("label-maps/payload.bin").read_bytes()
metadata = json.loads(root.joinpath("models.json").read_text())
source = root.joinpath("vendor/model.py").read_bytes()
source_digest = hashlib.sha256(source).hexdigest()
assert source_digest == metadata["source_sha256"]
assert hashlib.sha256(payload).hexdigest() == metadata["sha256"]
return value + metadata["offset"]
""",
"models.json": "{}",
},
)
payload = b"\x00\xff\r\nvendored model data"
package_root = Path(package.__file__).parent
(package_root / "label-maps").mkdir()
(package_root / "label-maps/payload.bin").write_bytes(payload)
module_path = package_root / "vendor/model.py"
module_bytes = (
module_path.read_bytes().replace(b"\r\n", b"\n").replace(b"\n", b"\r\n")
)
module_path.write_bytes(module_bytes)
(package_root / "models.json").write_bytes(
json.dumps(
{
"sha256": hashlib.sha256(payload).hexdigest(),
"source_sha256": hashlib.sha256(module_bytes).hexdigest(),
"offset": 7,
}
).encode()
+ b"\r\n"
)
entry = helper_package(
"fx_vendor_entry",
{
"__init__.py": """
import fx_vendor
from lancedb import function
@function(code=[fx_vendor])
def vendored(value: int) -> int:
return fx_vendor.predict(value)
"""
},
)
vendored = entry.vendored
first_digest = vendored.registration_request.artifact.digest
bundle = json.loads(artifact(vendored))
isolated = tmp_path / "isolated"
isolated.mkdir()
for name, content in bundle["files"].items():
target = isolated / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(content.encode("utf-8"))
for name, content in bundle["resources"].items():
target = isolated / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(base64.b64decode(content))
assert (isolated / "fx_vendor/models.json").read_bytes().endswith(b"\r\n")
result = subprocess.run(
[
sys.executable,
"-I",
"-c",
"import sys; sys.path.insert(0, sys.argv[1]); "
"from user_function import vendored; assert vendored(3) == 10",
str(isolated),
],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
(package_root / "models.json").write_text('{"offset": 8}')
assert (
function(code=[package])(
vendored._function
).registration_request.artifact.digest
!= first_digest
)
def test_code_packages_ship_as_a_bundle_and_version_with_their_source(
tmp_path, helper_package
):
@@ -672,14 +769,13 @@ def test_code_packages_ship_as_a_bundle_and_version_with_their_source(
first = define()
assert len(caught) == 1
warning = str(caught[0].message)
assert "fx_helpers/fixtures/data.json" in warning
assert "fx_helpers/fixtures/data.json" not in warning
assert "fx_helpers/not-a-module.py" in warning
assert warning.index("fx_helpers/fixtures/data.json") < warning.index(
"fx_helpers/not-a-module.py"
)
assert "remove 'fx_helpers' from code=" in warning
assert "pip=" in warning
assert "conda=" in warning
assert "safe relative names" in warning
assert "importable Python module names" in warning
resources = json.loads(artifact(first))["resources"]
assert base64.b64decode(resources["fx_helpers/fixtures/data.json"]) == b"{}"
assert "__pycache__" not in warning
assert ".pyc" not in warning
shipped = bundle_files(first)
@@ -732,7 +828,7 @@ def test_code_package_cache_artifacts_do_not_warn(helper_package):
]
def test_code_package_nonidentifier_resource_dir_warns_at_boundary(helper_package):
def test_code_package_resources_keep_nonidentifier_directories(helper_package):
resources = helper_package(
"fx_resources",
{
@@ -748,11 +844,9 @@ def test_code_package_nonidentifier_resource_dir_warns_at_boundary(helper_packag
def resource_boundary(value: int) -> int:
return value
assert len(caught) == 1
warning = str(caught[0].message)
assert "fx_resources/label-maps/" in warning
assert "remove 'fx_resources' from code=" in warning
assert "labels.json" not in warning
assert not caught
resources = json.loads(artifact(resource_boundary))["resources"]
assert base64.b64decode(resources["fx_resources/label-maps/labels.json"]) == b"{}"
assert sorted(bundle_files(resource_boundary)) == [
"fx_resources/__init__.py",
"user_function.py",