mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
Function-column declarations now preserve constants such as
`fn(video=col("mp4"), num_frames=lit(16))` without changing its Function
version. Constants travel in the application inputs; column bindings,
request input schemas, and dependency tracking contain only columns.
Changing a constant uses a new column binding rather than rebaking the
Function.
Persisted bindings with literals use metadata version 2, which older
readers reject. Column-only bindings retain version 1. Python preserves
scalar literal identity, including null and unsigned integers, and
rejects composed expressions or values outside the canonical literal
domain.
This SDK change requires matching server declaration and execution
support, after a dedicated SDK dependency update. Native validation
passed 48 Rust metadata tests, four request-transport cases and 57
rebuilt Python tests on `a42d680`; 32 pre-existing experimental tests
remained ignored. The test-assertion correction passed its exact
regression and strict Clippy on `53ff80a2`. On `993fae7d`, the three
remote metadata compatibility regressions, native Python public-export
test and strict Clippy passed. These review corrections use v3 for
unknown-version rejection, cover existing v2 literal bindings through
remote mutation guards, and export `LiteralInputBinding`. The unchanged
native implementation and verified extension are reused. Matching server
execution and original-workload acceptance are not established by this
PR.
Refs ENT-3667.
3057 lines
118 KiB
Python
3057 lines
118 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
"""Canonical Function values exchanged with LanceDB Enterprise services.
|
|
|
|
These immutable models contain client/wire state only. Catalog persistence,
|
|
environment bake, secret resolution, and execution are owned by LanceDB Enterprise.
|
|
``RefreshColumnResult`` is also the backend-neutral result of a local
|
|
expression-backed refresh job.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import builtins
|
|
import base64
|
|
import functools
|
|
import hashlib
|
|
import importlib.util
|
|
import inspect
|
|
import symtable
|
|
import json
|
|
import linecache
|
|
import math
|
|
import os
|
|
import re
|
|
import sys
|
|
import textwrap
|
|
import types
|
|
import warnings
|
|
from collections.abc import Mapping, Sequence
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
from typing import (
|
|
Annotated,
|
|
Any,
|
|
Callable,
|
|
Literal,
|
|
Optional,
|
|
TYPE_CHECKING,
|
|
Union,
|
|
get_args,
|
|
get_origin,
|
|
get_type_hints,
|
|
overload,
|
|
)
|
|
|
|
import pyarrow as pa
|
|
from packaging.requirements import Requirement
|
|
from pydantic import (
|
|
AfterValidator,
|
|
BaseModel,
|
|
ConfigDict,
|
|
Field,
|
|
conint,
|
|
field_validator,
|
|
model_validator,
|
|
)
|
|
|
|
from .schema import blob as _blob_field
|
|
from .schema import is_blob_v2_field as _is_blob_v2_field
|
|
from ._blob_reader import BlobReader
|
|
from ._lancedb import _validate_conda_requirement
|
|
from .secrets import EnvVarSecret
|
|
|
|
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
|
|
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
|
|
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
|
|
_GPU_CLASS_NAME = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$")
|
|
_GPU_CLASS_ERROR = (
|
|
"gpu_class must be a lowercase name of at most 63 letters, digits, or "
|
|
"hyphens, and cannot start or end with a hyphen"
|
|
)
|
|
|
|
|
|
def _validate_gpu_wire_marker(value: Any) -> bool:
|
|
if value is not True:
|
|
raise ValueError("runtime.gpu must be true")
|
|
return True
|
|
|
|
|
|
def _normalize_gpu_marker(value: bool) -> Optional[bool]:
|
|
if not isinstance(value, bool):
|
|
raise ValueError("gpu must be a boolean")
|
|
return True if value else None
|
|
|
|
|
|
def _validate_gpu_class_name(value: Any) -> str:
|
|
if not isinstance(value, str) or not _GPU_CLASS_NAME.fullmatch(value):
|
|
raise ValueError(_GPU_CLASS_ERROR)
|
|
return value
|
|
|
|
|
|
def _validate_batch_size(value: Any) -> Optional[int]:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
raise ValueError("batch_size must be an integer")
|
|
if not 1 <= value <= 2**32 - 1:
|
|
raise ValueError("batch_size must be between 1 and 4294967295")
|
|
return value
|
|
|
|
|
|
def _validate_max_concurrency(value: Any) -> Optional[int]:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
raise ValueError("max_concurrency must be an integer")
|
|
if not 1 <= value <= 2**32 - 1:
|
|
raise ValueError("max_concurrency must be between 1 and 4294967295")
|
|
return value
|
|
|
|
|
|
def _validate_row_timeout(value: Any) -> Optional[float]:
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
raise ValueError("row_timeout_seconds must be a number")
|
|
value = float(value)
|
|
if not math.isfinite(value) or value <= 0:
|
|
raise ValueError("row_timeout_seconds must be positive and finite")
|
|
return value
|
|
|
|
|
|
class _FrozenDict(dict):
|
|
def _immutable(self, *args, **kwargs):
|
|
raise TypeError("remote canonical values are immutable")
|
|
|
|
__setitem__ = _immutable
|
|
__delitem__ = _immutable
|
|
clear = _immutable
|
|
pop = _immutable
|
|
popitem = _immutable
|
|
setdefault = _immutable
|
|
update = _immutable
|
|
|
|
def __ior__(self, other):
|
|
self._immutable()
|
|
|
|
|
|
def _freeze_value(value):
|
|
if isinstance(value, Mapping):
|
|
return _FrozenDict({key: _freeze_value(child) for key, child in value.items()})
|
|
if isinstance(value, (list, tuple)):
|
|
return tuple(_freeze_value(child) for child in value)
|
|
return value
|
|
|
|
|
|
def _validate_literal(value):
|
|
if isinstance(value, float):
|
|
raise ValueError(
|
|
"floating-point Function literals are not part of the Slice 1 "
|
|
"canonical wire contract"
|
|
)
|
|
if isinstance(value, int) and not isinstance(value, bool):
|
|
if not -(2**63) <= value <= 2**64 - 1:
|
|
raise ValueError(
|
|
"Function integer literal is outside the canonical JSON range"
|
|
)
|
|
elif isinstance(value, Mapping):
|
|
if not all(isinstance(key, str) for key in value):
|
|
raise TypeError("Function literal object keys must be strings")
|
|
for child in value.values():
|
|
_validate_literal(child)
|
|
elif isinstance(value, (list, tuple)):
|
|
for child in value:
|
|
_validate_literal(child)
|
|
elif value is not None and not isinstance(value, (bool, str)):
|
|
raise TypeError(f"unsupported Function literal type: {type(value).__name__}")
|
|
return value
|
|
|
|
|
|
def _known_wire_value(value):
|
|
if isinstance(value, _RemoteValue):
|
|
return value._known_dict()
|
|
if isinstance(value, Mapping):
|
|
return {key: _known_wire_value(child) for key, child in value.items()}
|
|
if isinstance(value, (list, tuple)):
|
|
return [_known_wire_value(child) for child in value]
|
|
return value
|
|
|
|
|
|
class _RemoteValue(BaseModel):
|
|
model_config = ConfigDict(extra="ignore", frozen=True)
|
|
|
|
@model_validator(mode="after")
|
|
def _freeze_mappings(self):
|
|
for name, value in self.__dict__.items():
|
|
object.__setattr__(self, name, _freeze_value(value))
|
|
return self
|
|
|
|
@classmethod
|
|
def from_json(cls, payload: str):
|
|
return cls.model_validate_json(payload)
|
|
|
|
def _known_dict(self) -> dict[str, Any]:
|
|
known = {}
|
|
for name, field in self.__class__.model_fields.items():
|
|
value = getattr(self, name)
|
|
if value is None and not field.is_required():
|
|
continue
|
|
if not field.is_required():
|
|
default_factory = field.default_factory
|
|
if default_factory is not None and value == default_factory():
|
|
continue
|
|
if default_factory is None and value == field.default:
|
|
continue
|
|
known[name] = _known_wire_value(value)
|
|
return known
|
|
|
|
def _copy(self, *, update: Mapping[str, Any]):
|
|
update = {name: _freeze_value(value) for name, value in update.items()}
|
|
return self.model_copy(update=update)
|
|
|
|
def to_canonical_json(self) -> str:
|
|
return json.dumps(
|
|
self._known_dict(),
|
|
ensure_ascii=False,
|
|
allow_nan=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
)
|
|
|
|
|
|
class _OpenRemoteValue(_RemoteValue):
|
|
"""Forward-readable value whose extras stay out of canonical encoding."""
|
|
|
|
model_config = ConfigDict(extra="allow", frozen=True)
|
|
|
|
def _unknown_field_names(self) -> set[str]:
|
|
return set((self.__pydantic_extra__ or {}).keys())
|
|
|
|
|
|
class FunctionRetryPolicy(_RemoteValue):
|
|
"""Bound retries of selected errors inside one Enterprise Function call.
|
|
|
|
``max_attempts`` includes the first call. Matchers are alternatives;
|
|
``transient=True`` selects connection/timeouts, HTTP 429/503 and the
|
|
provider codes ``SlowDown`` and ``ServiceUnavailable``. Exception names
|
|
may be qualified, and subclasses and wrapped causes also match.
|
|
|
|
Backoff doubles with full jitter, bounded by ``max_backoff_seconds``.
|
|
Retry-After above that bound ends retries. A failed batch is retried as
|
|
one call: under ``skip``, exhaustion nulls every row in that call.
|
|
Completed calls are never repeated. Only opt in when repeating partial
|
|
work is safe; this does not make external side effects exactly once.
|
|
|
|
>>> policy = FunctionRetryPolicy(transient=True, max_attempts=3)
|
|
>>> policy.max_attempts
|
|
3
|
|
>>> disabled = FunctionRetryPolicy(max_attempts=1)
|
|
"""
|
|
|
|
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
|
|
max_attempts: Annotated[int, Field(strict=True)] = 3
|
|
initial_backoff_seconds: Annotated[float, Field(strict=True)] = 0.2
|
|
max_backoff_seconds: Annotated[float, Field(strict=True)] = 30.0
|
|
exception_types: tuple[str, ...] = ()
|
|
message_pattern: Optional[str] = None
|
|
error_codes: tuple[str, ...] = ()
|
|
transient: Annotated[bool, Field(strict=True)] = False
|
|
honor_retry_after: Annotated[bool, Field(strict=True)] = True
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_policy(self):
|
|
from ._lancedb import validate_function_retry_policy
|
|
|
|
validate_function_retry_policy(self.model_dump_json())
|
|
return self
|
|
|
|
|
|
class FunctionArtifact(_RemoteValue):
|
|
"""Content-addressed Python artifact identity."""
|
|
|
|
kind: str
|
|
digest: str
|
|
entrypoint: str
|
|
|
|
|
|
class FunctionArtifactContent(_RemoteValue):
|
|
"""Encoded artifact bytes uploaded during remote registration."""
|
|
|
|
encoding: str
|
|
data: str
|
|
|
|
|
|
class PythonAdapterSpec(_RemoteValue):
|
|
"""Internal scalar-callable to Arrow-batch adapter selection."""
|
|
|
|
kind: str
|
|
version: _UInt32
|
|
|
|
|
|
class FunctionArtifactRequest(_RemoteValue):
|
|
"""Source artifact uploaded while registering a Function."""
|
|
|
|
kind: str
|
|
digest: str
|
|
entrypoint: str
|
|
content: FunctionArtifactContent
|
|
adapter: PythonAdapterSpec
|
|
|
|
|
|
class FunctionParameter(_RemoteValue):
|
|
name: str
|
|
arrow_type: str
|
|
nullable: bool
|
|
blob_access: Optional[Literal["bytes", "reader"]] = None
|
|
|
|
|
|
class FunctionResultField(_OpenRemoteValue):
|
|
name: str
|
|
arrow_type: str
|
|
nullable: bool
|
|
|
|
|
|
class FunctionOutput(_OpenRemoteValue):
|
|
"""Scalar or ordered named-struct output; unknown kinds remain decodable.
|
|
|
|
A scalar output may be nullable. A named-struct output has no top-level
|
|
nullability; its fields carry their own flags.
|
|
"""
|
|
|
|
kind: str
|
|
arrow_type: Optional[str] = None
|
|
nullable: Optional[bool] = None
|
|
fields: tuple[FunctionResultField, ...] = ()
|
|
|
|
|
|
class SecretReference(_RemoteValue):
|
|
"""Where a Secret lives, carried as its parts rather than as one string.
|
|
|
|
A joined id would need a delimiter, and a delimiter has to be excluded from
|
|
every name and segment forever, agreed on by both sides, and re-agreed each
|
|
time either grows a new way to be configured. Naming the parts settles all
|
|
of that: nothing here is parsed, so nothing can parse two ways.
|
|
"""
|
|
|
|
name: str
|
|
namespace_path: tuple[str, ...] = ()
|
|
|
|
|
|
class SecretBinding(_RemoteValue):
|
|
"""How a Secret reaches the Function that binds it.
|
|
|
|
One list rather than a field per delivery mode: a binding is the concept,
|
|
and how it arrives is a property of one. ``kind`` is open, so a binding a
|
|
newer service introduces decodes here instead of failing the whole
|
|
FunctionVersion.
|
|
"""
|
|
|
|
kind: str
|
|
variable: Optional[str] = None
|
|
secret_ref: Optional[SecretReference] = None
|
|
|
|
|
|
class FunctionSignature(_RemoteValue):
|
|
"""Ordered inputs, output, and initialization fields of a Function.
|
|
|
|
``initialization`` lists the fields of the single row each Function
|
|
instance is created with. Values come from each binding, not from the
|
|
Function version; the tuple is empty when the Function takes none.
|
|
"""
|
|
|
|
inputs: tuple[FunctionParameter, ...]
|
|
output: FunctionOutput
|
|
initialization: tuple[FunctionParameter, ...] = ()
|
|
|
|
|
|
class PythonEnvironmentSpec(_RemoteValue):
|
|
"""One LanceDB Enterprise-managed Python environment source."""
|
|
|
|
kind: str
|
|
packages: tuple[str, ...] = ()
|
|
channels: tuple[str, ...] = ()
|
|
path: Optional[str] = None
|
|
modules: tuple[str, ...] = ()
|
|
image: Optional[str] = None
|
|
system_packages: tuple[str, ...] = ()
|
|
|
|
|
|
_HUGGING_FACE_MODEL_REPO = re.compile(
|
|
r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$"
|
|
)
|
|
_HUGGING_FACE_REVISION = re.compile(r"^[0-9a-f]{40}$")
|
|
|
|
|
|
class HuggingFaceModel(_RemoteValue):
|
|
"""One immutable Hugging Face model snapshot required by a Function.
|
|
|
|
``revision`` is a full 40-character lowercase commit SHA, not a branch or
|
|
tag. A Function may declare each repository once; separate versions of the
|
|
same repository use separate Function versions. The service prepares the
|
|
snapshot before initialization, so the model can load without a revision
|
|
argument or a Hugging Face Hub request::
|
|
|
|
@function(
|
|
pip=["sentence-transformers==3.3.1"],
|
|
models=[HuggingFaceModel(
|
|
repo="sentence-transformers/all-MiniLM-L6-v2",
|
|
revision="1110a243fdf4706b3f48f1d95db1a4f5529b4d41",
|
|
)],
|
|
)
|
|
class Embed:
|
|
def __init__(self):
|
|
from sentence_transformers import SentenceTransformer
|
|
|
|
self.model = SentenceTransformer(
|
|
"sentence-transformers/all-MiniLM-L6-v2", device="cpu"
|
|
)
|
|
|
|
def __call__(self, text: str) -> list[float]:
|
|
return self.model.encode(text).tolist()
|
|
"""
|
|
|
|
repo: str = Field(
|
|
pattern=r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$"
|
|
)
|
|
revision: str
|
|
|
|
@field_validator("repo")
|
|
@classmethod
|
|
def _validate_repo(cls, value: str) -> str:
|
|
if not _HUGGING_FACE_MODEL_REPO.fullmatch(value) or any(
|
|
part in {".", ".."} for part in value.split("/")
|
|
):
|
|
raise ValueError("model repo must be a Hugging Face owner/name")
|
|
return value
|
|
|
|
@field_validator("revision", mode="before")
|
|
@classmethod
|
|
def _validate_revision(cls, value: Any) -> str:
|
|
if not isinstance(value, str) or not _HUGGING_FACE_REVISION.fullmatch(value):
|
|
raise ValueError(
|
|
"model revision must be a full 40-character lowercase commit SHA"
|
|
)
|
|
return value
|
|
|
|
|
|
class PythonRuntimeSpec(_RemoteValue):
|
|
"""Remote runtime definition with environment values.
|
|
|
|
V1 supports ``kind="python"``. Newer runtime kinds remain readable, while
|
|
their unknown payload fields are intentionally not retained by the client.
|
|
``kind="python_v2"`` requires ``gpu=True`` and may name an operator-defined
|
|
``gpu_class``.
|
|
"""
|
|
|
|
kind: str
|
|
python_version: Optional[str] = None
|
|
environment: Optional[PythonEnvironmentSpec] = None
|
|
env: Optional[Mapping[str, str]] = None
|
|
gpu: Optional[bool] = None
|
|
gpu_class: Optional[str] = None
|
|
|
|
@model_validator(mode="before")
|
|
@classmethod
|
|
def _discard_unknown_runtime_payload(cls, value):
|
|
if isinstance(value, Mapping):
|
|
kind = value.get("kind")
|
|
if isinstance(kind, str) and kind not in {"python", "python_v2"}:
|
|
return {"kind": kind}
|
|
return value
|
|
|
|
@field_validator("gpu", mode="before")
|
|
@classmethod
|
|
def _validate_gpu_marker(cls, value):
|
|
if value is None:
|
|
return None
|
|
return _validate_gpu_wire_marker(value)
|
|
|
|
@field_validator("gpu_class", mode="before")
|
|
@classmethod
|
|
def _validate_gpu_class(cls, value):
|
|
if value is None:
|
|
return None
|
|
return _validate_gpu_class_name(value)
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_runtime_kind(self):
|
|
if self.kind == "python":
|
|
if self.python_version is None:
|
|
raise ValueError("python runtime requires python_version")
|
|
if self.environment is None:
|
|
raise ValueError("python runtime requires environment")
|
|
if self.gpu is not None:
|
|
raise ValueError("python runtime with gpu requires kind='python_v2'")
|
|
if self.gpu_class is not None:
|
|
raise ValueError(
|
|
"python runtime with gpu_class requires kind='python_v2'"
|
|
)
|
|
elif self.kind == "python_v2":
|
|
if self.python_version is None:
|
|
raise ValueError("python_v2 runtime requires python_version")
|
|
if self.environment is None:
|
|
raise ValueError("python_v2 runtime requires environment")
|
|
if self.gpu is None:
|
|
raise ValueError("python_v2 runtime requires gpu")
|
|
else:
|
|
object.__setattr__(self, "python_version", None)
|
|
object.__setattr__(self, "environment", None)
|
|
object.__setattr__(self, "env", None)
|
|
object.__setattr__(self, "gpu", None)
|
|
object.__setattr__(self, "gpu_class", None)
|
|
return self
|
|
|
|
@property
|
|
def digest(self) -> str:
|
|
"""SHA-256 identity of the declared runtime, before building an image.
|
|
|
|
Covers Python version, environment sources, environment variables, and
|
|
GPU settings. Package and system-package order is ignored; conda channel
|
|
order is preserved. Compare with ``runtime_digest`` on
|
|
[FunctionVersion][lancedb.functions.FunctionVersion].
|
|
This identifies build inputs, not the resolved contents of unpinned
|
|
dependencies or the deployment's default base image.
|
|
"""
|
|
if self.kind not in {"python", "python_v2"}:
|
|
raise ValueError(f"cannot digest unknown runtime kind {self.kind!r}")
|
|
value = self._known_dict()
|
|
if not value.get("env"):
|
|
value.pop("env", None)
|
|
environment = value["environment"]
|
|
for field in ("packages", "system_packages"):
|
|
if field in environment:
|
|
environment[field] = sorted(environment[field])
|
|
encoded = json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
allow_nan=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
return f"sha256:{hashlib.sha256(encoded).hexdigest()}"
|
|
|
|
|
|
class FunctionImage(_RemoteValue):
|
|
"""A complete OCI Function image identified by its exact manifest digest."""
|
|
|
|
manifest_digest: str
|
|
descriptor: Mapping[str, Any]
|
|
source: bool
|
|
models: tuple[HuggingFaceModel, ...] = ()
|
|
|
|
|
|
def _validate_object_version(value: str) -> str:
|
|
if int(value) > 2**64 - 1:
|
|
raise ValueError("Function version exceeds uint64")
|
|
return value
|
|
|
|
|
|
_ObjectVersion = Annotated[
|
|
str,
|
|
Field(strict=True, pattern=r"^[1-9][0-9]*$"),
|
|
AfterValidator(_validate_object_version),
|
|
]
|
|
|
|
|
|
class FunctionVersion(_RemoteValue):
|
|
"""A pinned object revision, independent of its executable image digest.
|
|
|
|
Unknown top-level fields from newer servers are accepted for forward
|
|
decoding, but they are not exposed on this value and are omitted from
|
|
``to_canonical_json()``. Known fields retain their complete values, while
|
|
unknown fields inside the open ``image.descriptor`` mapping remain
|
|
available.
|
|
|
|
``name`` and ``namespace_path`` are the Function's parts, as a
|
|
[SecretReference][lancedb.functions.SecretReference] carries a Secret's;
|
|
they are what [get_function][lancedb.db.DBConnection.get_function] takes.
|
|
An empty ``namespace_path`` is the root namespace.
|
|
|
|
``source_digest`` identifies the registered source artifact, independently
|
|
of the built ``image``. Compare it with
|
|
``definition.registration_request.artifact.digest`` to check whether a local
|
|
[FunctionDefinition][lancedb.functions.FunctionDefinition] has the same source.
|
|
It does not cover dependencies, environment variables, Python version, or
|
|
GPU settings. Equal source digests alone do not mean a version can be reused.
|
|
It is ``None`` when the server does not provide a source digest.
|
|
|
|
``runtime_digest`` identifies the declared environment inputs. Compare it
|
|
with ``definition.registration_request.runtime.digest`` to check whether
|
|
the local runtime matches. It is ``None`` for images built before runtime
|
|
identity was exposed. A missing digest cannot establish a match. Reusing a
|
|
version also requires matching the source and all other registration inputs,
|
|
including signature, Secret bindings, models, and execution controls.
|
|
|
|
``gpu_class`` is the operator-defined GPU pool fixed into this Function
|
|
version, or ``None`` when registration used the deployment default.
|
|
|
|
``batch_size``, ``max_concurrency``, ``row_timeout_seconds``, and
|
|
``result_stability`` retain the execution controls reported by the
|
|
catalog. They are ``None`` for versions registered before the
|
|
corresponding declaration.
|
|
"""
|
|
|
|
name: str
|
|
namespace_path: tuple[str, ...] = ()
|
|
object_id: str
|
|
location: str
|
|
version: _ObjectVersion
|
|
image: FunctionImage
|
|
source_digest: Optional[str] = None
|
|
runtime_digest: Optional[str] = None
|
|
signature: FunctionSignature
|
|
secret_bindings: tuple[SecretBinding, ...] = ()
|
|
created_at_millis: int
|
|
metadata: Mapping[str, str]
|
|
disabled: bool
|
|
gpu_class: Optional[str] = None
|
|
batch_size: Optional[int] = None
|
|
max_concurrency: Optional[int] = None
|
|
row_timeout_seconds: Optional[float] = None
|
|
result_stability: Optional[str] = None
|
|
|
|
@field_validator("batch_size", mode="before")
|
|
@classmethod
|
|
def _validate_batch_size(cls, value):
|
|
return _validate_batch_size(value)
|
|
|
|
@field_validator("max_concurrency", mode="before")
|
|
@classmethod
|
|
def _validate_max_concurrency(cls, value):
|
|
return _validate_max_concurrency(value)
|
|
|
|
@field_validator("row_timeout_seconds", mode="before")
|
|
@classmethod
|
|
def _validate_row_timeout(cls, value):
|
|
return _validate_row_timeout(value)
|
|
|
|
def __call__(self, **arguments: Any) -> FunctionApplication:
|
|
"""Bind this exact version to named table columns and constant inputs.
|
|
|
|
Every input must be a direct [lancedb.col][lancedb.expr.col]
|
|
reference or [lancedb.lit][lancedb.expr.lit] constant. Constants support
|
|
integers, strings, booleans, and null; floating-point, binary and
|
|
temporal literals are not in the canonical Function input contract.
|
|
The Enterprise service must support literal bindings and validates
|
|
constants against this exact version's input signature. To change a
|
|
constant, redeclare the column using the same Function version.
|
|
|
|
A Function that declares initialization fields takes their
|
|
values as further keyword arguments; they are constants of this
|
|
binding, fixed for every row, and every instance the binding runs is
|
|
created with them. A non-nullable field without a default must be
|
|
given; an omitted nullable field is null, and the Function's own
|
|
default applies to a null value.
|
|
|
|
The returned application is immutable and retains a named-struct
|
|
output as one binding, so every row's sibling values come from one
|
|
logical Function evaluation. Map result fields to table columns with
|
|
[FunctionApplication.rename][lancedb.functions.FunctionApplication.rename],
|
|
then pass the application to
|
|
[Table.add_columns][lancedb.table.Table.add_columns].
|
|
|
|
Examples
|
|
--------
|
|
>>> from lancedb import col
|
|
>>> application = function( # doctest: +SKIP
|
|
... title=col("title"),
|
|
... body=col("body"),
|
|
... ).rename(columns={
|
|
... "normalized_text": "search_text",
|
|
... "token_count": "search_token_count",
|
|
... })
|
|
>>> table.add_columns(application) # doctest: +SKIP
|
|
>>> table.add_columns({ # doctest: +SKIP
|
|
... "embedding": embed(text=col("body"), model="small", dimensions=512)
|
|
... })
|
|
"""
|
|
from lancedb.expr import Expr, _NOT_LITERAL
|
|
|
|
parameters = tuple(parameter.name for parameter in self.signature.inputs)
|
|
fields = {field.name: field for field in self.signature.initialization}
|
|
missing = [parameter for parameter in parameters if parameter not in arguments]
|
|
missing_initialization = [
|
|
name
|
|
for name, field in fields.items()
|
|
if not field.nullable and name not in arguments
|
|
]
|
|
unknown = sorted(set(arguments) - set(parameters) - set(fields))
|
|
if missing or missing_initialization or unknown:
|
|
details = []
|
|
if missing:
|
|
details.append(f"missing inputs: {missing!r}")
|
|
if missing_initialization:
|
|
details.append(f"missing initialization: {missing_initialization!r}")
|
|
if unknown:
|
|
details.append(f"unknown arguments: {unknown!r}")
|
|
raise TypeError("invalid Function arguments (" + "; ".join(details) + ")")
|
|
|
|
bindings = []
|
|
for parameter in parameters:
|
|
value = arguments[parameter]
|
|
if isinstance(value, Expr) and value._literal_value is not _NOT_LITERAL:
|
|
bindings.append(
|
|
ApplicationInput(
|
|
parameter=parameter,
|
|
kind="literal",
|
|
value=value._literal_value,
|
|
)
|
|
)
|
|
elif isinstance(value, Expr) and value._column_path is not None:
|
|
bindings.append(
|
|
ApplicationInput(
|
|
parameter=parameter,
|
|
kind="column",
|
|
value={"path": value._column_path},
|
|
)
|
|
)
|
|
else:
|
|
raise TypeError(
|
|
f"Function input {parameter!r} must be a direct col(...) reference "
|
|
"or lit(...) constant"
|
|
)
|
|
initialization = {}
|
|
for name in fields:
|
|
if name in arguments:
|
|
initialization[name] = _initialization_value(name, arguments[name])
|
|
return FunctionApplication(
|
|
function=FunctionVersionRef(
|
|
name=self.name,
|
|
namespace_path=self.namespace_path,
|
|
object_id=self.object_id,
|
|
location=self.location,
|
|
version=self.version,
|
|
manifest_digest=self.image.manifest_digest,
|
|
),
|
|
inputs=tuple(bindings),
|
|
output=self.signature.output,
|
|
initialization=initialization,
|
|
)
|
|
|
|
|
|
def _initialization_value(name: str, value: Any) -> Any:
|
|
"""One initialization value as plain JSON; the service checks its type."""
|
|
from lancedb.expr import Expr
|
|
|
|
def plain(value):
|
|
if isinstance(value, Expr):
|
|
raise TypeError(
|
|
f"initialization field {name!r} takes a constant, not a column "
|
|
"expression; every row of a binding shares one initialization"
|
|
)
|
|
if value is None or type(value) in (bool, int, str):
|
|
return value
|
|
if type(value) is float:
|
|
if not math.isfinite(value):
|
|
raise ValueError(
|
|
f"initialization field {name!r} must be finite, got {value!r}"
|
|
)
|
|
return value
|
|
if isinstance(value, (list, tuple)):
|
|
return [plain(child) for child in value]
|
|
if isinstance(value, Mapping) and all(isinstance(key, str) for key in value):
|
|
return {key: plain(child) for key, child in value.items()}
|
|
raise TypeError(
|
|
f"initialization field {name!r} takes booleans, numbers, strings, "
|
|
f"lists, and string-keyed mappings, got {type(value).__name__}"
|
|
)
|
|
|
|
return plain(value)
|
|
|
|
|
|
class FunctionRegistrationRequest(_RemoteValue):
|
|
"""Stable remote registration envelope produced by :func:`function`.
|
|
|
|
Credential values deliberately have no field here. The only secret-shaped
|
|
thing a client sends is ``secret_bindings``: the name of a Secret the
|
|
database already holds, which the remote service resolves at execution.
|
|
"""
|
|
|
|
name: str
|
|
artifact: FunctionArtifactRequest
|
|
signature: FunctionSignature
|
|
runtime: PythonRuntimeSpec
|
|
batch_size: Optional[int] = None
|
|
max_concurrency: Optional[int] = None
|
|
row_timeout_seconds: Optional[float] = None
|
|
retry_policy: Optional[FunctionRetryPolicy] = None
|
|
secret_bindings: tuple[SecretBinding, ...] = ()
|
|
models: tuple[HuggingFaceModel, ...] = ()
|
|
|
|
@field_validator("batch_size", mode="before")
|
|
@classmethod
|
|
def _validate_batch_size(cls, value):
|
|
return _validate_batch_size(value)
|
|
|
|
@field_validator("max_concurrency", mode="before")
|
|
@classmethod
|
|
def _validate_max_concurrency(cls, value):
|
|
return _validate_max_concurrency(value)
|
|
|
|
@field_validator("row_timeout_seconds", mode="before")
|
|
@classmethod
|
|
def _validate_row_timeout(cls, value):
|
|
return _validate_row_timeout(value)
|
|
|
|
|
|
class FunctionVersionRef(_OpenRemoteValue):
|
|
name: str
|
|
namespace_path: tuple[str, ...] = ()
|
|
object_id: str
|
|
location: str
|
|
version: _ObjectVersion
|
|
manifest_digest: str
|
|
|
|
|
|
class ApplicationInput(_OpenRemoteValue):
|
|
"""One parameter value.
|
|
|
|
Slice 1 freezes integers, strings, booleans, nulls, arrays, and objects.
|
|
Floating-point literal encoding is deferred until Python authoring is
|
|
introduced with a language-neutral numeric representation.
|
|
"""
|
|
|
|
parameter: str
|
|
kind: str
|
|
value: Any
|
|
|
|
@field_validator("value")
|
|
@classmethod
|
|
def _validate_value(cls, value):
|
|
return _validate_literal(value)
|
|
|
|
|
|
class FunctionApplication(_OpenRemoteValue):
|
|
"""Immutable pre-declaration application of an exact Function version.
|
|
|
|
A named-struct output remains one application through table
|
|
declaration and execution.
|
|
[FunctionApplication.rename][lancedb.functions.FunctionApplication.rename]
|
|
records the result-field to table-column mapping without splitting sibling
|
|
outputs into separate UDF calls.
|
|
"""
|
|
|
|
function: FunctionVersionRef
|
|
inputs: tuple[ApplicationInput, ...]
|
|
output: FunctionOutput
|
|
columns: Mapping[str, str] = Field(default_factory=dict)
|
|
# Transport JSON: the service validates it against the Function's
|
|
# initialization schema and persists the Arrow row in the binding, so it is
|
|
# never hashed and floating-point values are allowed.
|
|
initialization: Mapping[str, Any] = Field(default_factory=dict)
|
|
|
|
def _known_dict(self) -> dict[str, Any]:
|
|
value = super()._known_dict()
|
|
for name in self._unknown_field_names():
|
|
value.pop(name, None)
|
|
return value
|
|
|
|
def _ensure_declarable(self) -> None:
|
|
unknown = {f"application.{name}" for name in self._unknown_field_names()}
|
|
unknown.update(
|
|
f"function.{name}" for name in self.function._unknown_field_names()
|
|
)
|
|
for index, input_value in enumerate(self.inputs):
|
|
unknown.update(
|
|
f"inputs[{index}].{name}" for name in input_value._unknown_field_names()
|
|
)
|
|
unknown.update(f"output.{name}" for name in self.output._unknown_field_names())
|
|
for index, field in enumerate(self.output.fields):
|
|
unknown.update(
|
|
f"output.fields[{index}].{name}"
|
|
for name in field._unknown_field_names()
|
|
)
|
|
if unknown:
|
|
raise ValueError(
|
|
"Function application contains fields from a newer contract: "
|
|
f"{sorted(unknown)!r}"
|
|
)
|
|
|
|
def rename(self, *, columns: Mapping[str, str]) -> FunctionApplication:
|
|
"""Return a copy with result-field to table-column aliases."""
|
|
if self.output.kind != "named_struct":
|
|
raise ValueError("rename(columns=...) requires a named-struct application")
|
|
result_fields = {field.name for field in self.output.fields}
|
|
unknown = set(columns) - result_fields
|
|
if unknown:
|
|
raise ValueError(f"unknown Function result fields: {sorted(unknown)!r}")
|
|
merged = dict(self.columns)
|
|
merged.update(columns)
|
|
destinations = tuple(
|
|
merged.get(field.name, field.name) for field in self.output.fields
|
|
)
|
|
if len(set(destinations)) != len(destinations):
|
|
raise ValueError("FunctionApplication rename destinations must be unique")
|
|
return self._copy(update={"columns": merged})
|
|
|
|
|
|
class InputBinding(_RemoteValue):
|
|
parameter: str
|
|
field_id: _Int32
|
|
field_path: str
|
|
arrow_type: str
|
|
nullable: bool
|
|
|
|
|
|
class LiteralInputBinding(_RemoteValue):
|
|
"""Constant supplied to a Function parameter for every input row."""
|
|
|
|
parameter: str
|
|
value: Any
|
|
|
|
@field_validator("value")
|
|
@classmethod
|
|
def _validate_value(cls, value):
|
|
return _validate_literal(value)
|
|
|
|
|
|
class OutputMapping(_RemoteValue):
|
|
"""One stable result-field mapping."""
|
|
|
|
result_field: str
|
|
output_name: str
|
|
output_field_id: _Int32
|
|
output_ordinal: _UInt32
|
|
arrow_type: str
|
|
nullable: bool
|
|
|
|
|
|
class AssignmentMapping(_RemoteValue):
|
|
"""Internal physical column preserving flattened struct validity."""
|
|
|
|
output_name: str
|
|
output_field_id: _Int32
|
|
|
|
|
|
class FunctionBinding(_RemoteValue):
|
|
"""Immutable Function binding persisted by the Enterprise table service."""
|
|
|
|
binding_id: str
|
|
function: FunctionVersionRef
|
|
inputs: tuple[InputBinding, ...]
|
|
literal_inputs: tuple[LiteralInputBinding, ...] = ()
|
|
outputs: tuple[OutputMapping, ...]
|
|
assignment: Optional[AssignmentMapping] = None
|
|
input_schema: Optional[Mapping[str, Any]] = None
|
|
output_schema: Optional[Mapping[str, Any]] = None
|
|
initialization: Optional[str] = None
|
|
"""Standard base64 of the validated one-row Arrow IPC initialization stream."""
|
|
|
|
@model_validator(mode="after")
|
|
def _validate_input_parameters(self):
|
|
parameters = [input.parameter for input in (*self.inputs, *self.literal_inputs)]
|
|
if len(parameters) != len(set(parameters)):
|
|
raise ValueError("Function binding input parameters must be unique")
|
|
return self
|
|
|
|
def initialization_row(self) -> Optional[dict[str, Any]]:
|
|
"""The initialization values this binding creates instances with."""
|
|
if self.initialization is None:
|
|
return None
|
|
reader = pa.ipc.open_stream(base64.b64decode(self.initialization))
|
|
return reader.read_all().to_pylist()[0]
|
|
|
|
|
|
class RefreshColumnResult(_RemoteValue):
|
|
"""Terminal result of an expression-backed or Function-backed refresh Job.
|
|
|
|
LanceDB Embedded jobs produce this value in process. LanceDB Enterprise
|
|
decodes the same value from the durable server-job terminal payload.
|
|
|
|
Row counters describe logical rows planned for this refresh, not user-code
|
|
calls or retry attempts. ``rows_assigned`` counts rows successfully written
|
|
with an output value, ``rows_failed`` counts rows whose computation failed,
|
|
and ``rows_remaining`` counts planned rows neither assigned nor terminally
|
|
skipped. Rows already counted in ``rows_failed`` are covered and are not
|
|
counted again in ``rows_remaining``.
|
|
|
|
For example, a refresh over 2,001 logical rows that assigns 1,891 values and
|
|
fails 110 rows reports ``rows_assigned=1,891``, ``rows_failed=110``, and
|
|
``rows_remaining=0``. Its failure-rate denominator is those 2,001 logical
|
|
rows, not the number of execution attempts.
|
|
"""
|
|
|
|
rows_assigned: _UInt64
|
|
"""Number of logical rows successfully assigned an output value by this
|
|
refresh. Each row counts once, regardless of retry attempts, and failed rows
|
|
are not included."""
|
|
|
|
rows_failed: _UInt64
|
|
"""Number of logical rows whose computation failed. Rows deferred because
|
|
an input was not ready are not failures."""
|
|
|
|
rows_remaining: _UInt64
|
|
"""Number of planned logical rows neither assigned nor terminally skipped.
|
|
Rows already counted in ``rows_failed`` are covered and are not counted
|
|
again."""
|
|
|
|
source_version: _UInt64
|
|
"""Exact table version the refresh read."""
|
|
|
|
published_version: Optional[_UInt64] = None
|
|
"""Table version made visible by the refresh, when one was published."""
|
|
|
|
@property
|
|
def rows_filled(self) -> int:
|
|
"""Deprecated compatibility alias for :attr:`rows_assigned`.
|
|
|
|
The value is the same successful logical-row count; it does not count
|
|
execution attempts.
|
|
"""
|
|
return self.rows_assigned
|
|
|
|
@property
|
|
def version(self) -> Optional[int]:
|
|
"""Deprecated compatibility alias for :attr:`published_version`."""
|
|
return self.published_version
|
|
|
|
|
|
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
# Digest-pinned only; the server checks the host and repository grammar.
|
|
_IMAGE_REFERENCE = re.compile(r"^[^/@\s]+/[^@\s]+@sha256:[0-9a-f]{64}$")
|
|
|
|
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
|
|
_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
|
|
_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
|
|
_NESTED_BLOB_COLLECTION_ERROR = (
|
|
"unsupported Arrow type for Function signature: Blob v2 fields nested under "
|
|
"collection types are not supported"
|
|
)
|
|
|
|
|
|
_GRAMMAR_PRIMITIVES = (
|
|
(pa.bool_(), "bool"),
|
|
(pa.int8(), "int8"),
|
|
(pa.int16(), "int16"),
|
|
(pa.int32(), "int32"),
|
|
(pa.int64(), "int64"),
|
|
(pa.uint8(), "uint8"),
|
|
(pa.uint16(), "uint16"),
|
|
(pa.uint32(), "uint32"),
|
|
(pa.uint64(), "uint64"),
|
|
(pa.float16(), "float16"),
|
|
(pa.float32(), "float32"),
|
|
(pa.float64(), "float64"),
|
|
(pa.string(), "utf8"),
|
|
(pa.large_string(), "large_utf8"),
|
|
(pa.binary(), "binary"),
|
|
(pa.large_binary(), "large_binary"),
|
|
(pa.date32(), "date32"),
|
|
(pa.date64(), "date64"),
|
|
)
|
|
|
|
|
|
def _canonical_arrow_type(
|
|
data_type: pa.DataType, *, collection_blobs: bool = False
|
|
) -> str:
|
|
"""The compact Function grammar, or canonical exact JSON for nested types."""
|
|
grammar = _grammar_arrow_type(data_type)
|
|
if grammar is not None:
|
|
return grammar
|
|
exact = _exact_arrow_type(data_type, collection_blobs=collection_blobs)
|
|
return json.dumps(exact, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
|
|
|
|
def _grammar_arrow_type(data_type: pa.DataType) -> Optional[str]:
|
|
for candidate, name in _GRAMMAR_PRIMITIVES:
|
|
if data_type == candidate:
|
|
return name
|
|
if pa.types.is_list(data_type) or pa.types.is_large_list(data_type):
|
|
item = _grammar_list_item(data_type)
|
|
if item is None:
|
|
return None
|
|
prefix = "list" if pa.types.is_list(data_type) else "large_list"
|
|
return f"{prefix}<{item}>"
|
|
if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0:
|
|
item = _grammar_list_item(data_type)
|
|
if item is not None:
|
|
return f"fixed_size_list<{item}, {data_type.list_size}>"
|
|
return None
|
|
|
|
|
|
def _grammar_list_item(data_type: pa.DataType) -> Optional[str]:
|
|
"""The grammar names only the item type; it always means a non-nullable
|
|
child called `item`, so other child properties require exact JSON."""
|
|
child = data_type.value_field
|
|
if child.name != "item" or child.nullable or child.metadata:
|
|
return None
|
|
return _grammar_arrow_type(child.type)
|
|
|
|
|
|
def _validate_exact_arrow_field(field: pa.Field) -> None:
|
|
if not field.name:
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: field names "
|
|
"must not be empty"
|
|
)
|
|
if _is_blob_v2_field(field):
|
|
if not _has_supported_blob_v2_layout(field):
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: lance.blob.v2 "
|
|
f"requires a supported Blob storage layout, got {field}"
|
|
)
|
|
metadata = {
|
|
(key.decode() if isinstance(key, bytes) else key): (
|
|
value.decode() if isinstance(value, bytes) else value
|
|
)
|
|
for key, value in (field.metadata or {}).items()
|
|
}
|
|
if metadata and metadata != {
|
|
_ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME
|
|
}:
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: lance.blob.v2 "
|
|
"field metadata must contain only its canonical extension marker"
|
|
)
|
|
elif field.metadata:
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: field metadata "
|
|
f"is not supported, got {field}"
|
|
)
|
|
|
|
|
|
def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
|
|
data_type = field.type
|
|
if isinstance(data_type, pa.ExtensionType):
|
|
data_type = data_type.storage_type
|
|
if not pa.types.is_struct(data_type):
|
|
return False
|
|
|
|
fields = tuple(data_type)
|
|
|
|
def matches(spec, compare_nullable) -> bool:
|
|
return len(fields) == len(spec) and all(
|
|
actual.name == name
|
|
and actual.type == expected_type
|
|
and (not check_nullable or actual.nullable == nullable)
|
|
for actual, (name, expected_type, nullable), check_nullable in zip(
|
|
fields, spec, compare_nullable
|
|
)
|
|
)
|
|
|
|
logical_minimal = (
|
|
("data", pa.large_binary(), True),
|
|
("uri", pa.utf8(), True),
|
|
)
|
|
logical_full = logical_minimal + (
|
|
("position", pa.uint64(), True),
|
|
("size", pa.uint64(), True),
|
|
)
|
|
prepared = (
|
|
("kind", pa.uint8(), True),
|
|
("data", pa.large_binary(), True),
|
|
("uri", pa.utf8(), True),
|
|
("blob_id", pa.uint32(), True),
|
|
("blob_size", pa.uint64(), True),
|
|
("position", pa.uint64(), True),
|
|
)
|
|
descriptor = (
|
|
("kind", pa.uint8(), False),
|
|
("position", pa.uint64(), False),
|
|
("size", pa.uint64(), False),
|
|
("blob_id", pa.uint32(), False),
|
|
("blob_uri", pa.utf8(), False),
|
|
)
|
|
return (
|
|
matches(logical_minimal, (True, True))
|
|
or matches(logical_full, (True, True, False, False))
|
|
or matches(prepared, (True,) * len(prepared))
|
|
or matches(descriptor, (False,) * len(descriptor))
|
|
)
|
|
|
|
|
|
def _canonical_arrow_field(field: pa.Field, *, collection_blobs: bool = False) -> str:
|
|
"""``collection_blobs`` admits Blob v2 under a list, which only outputs support."""
|
|
_validate_exact_arrow_field(field)
|
|
if _is_blob_v2_field(field):
|
|
return _FUNCTION_BLOB_V2_TYPE
|
|
return _canonical_arrow_type(field.type, collection_blobs=collection_blobs)
|
|
|
|
|
|
def _blob_storage_type(field: pa.Field) -> pa.DataType:
|
|
data_type = field.type
|
|
if isinstance(data_type, pa.ExtensionType):
|
|
return data_type.storage_type
|
|
return data_type
|
|
|
|
|
|
def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
|
|
storage = _blob_storage_type(field)
|
|
if not pa.types.is_struct(storage):
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: lance.blob.v2 "
|
|
"requires struct storage"
|
|
)
|
|
return {
|
|
"type": "struct",
|
|
"fields": [
|
|
{
|
|
"name": child.name,
|
|
"nullable": child.nullable,
|
|
"type": _exact_arrow_type(child.type),
|
|
}
|
|
for child in storage
|
|
],
|
|
}
|
|
|
|
|
|
def _data_type_has_blob_v2(data_type: pa.DataType) -> bool:
|
|
if pa.types.is_struct(data_type):
|
|
return any(
|
|
_is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
|
|
for field in data_type
|
|
)
|
|
if (
|
|
pa.types.is_list(data_type)
|
|
or pa.types.is_large_list(data_type)
|
|
or pa.types.is_fixed_size_list(data_type)
|
|
):
|
|
field = data_type.value_field
|
|
return _is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
|
|
if pa.types.is_map(data_type):
|
|
return any(
|
|
_is_blob_v2_field(field) or _data_type_has_blob_v2(field.type)
|
|
for field in (data_type.key_field, data_type.item_field)
|
|
)
|
|
return False
|
|
|
|
|
|
def _exact_arrow_field(
|
|
field: pa.Field,
|
|
*,
|
|
inside_collection: bool = False,
|
|
collection_blobs: bool = False,
|
|
) -> dict[str, Any]:
|
|
_validate_exact_arrow_field(field)
|
|
if _is_blob_v2_field(field):
|
|
if inside_collection and not collection_blobs:
|
|
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
|
|
return {
|
|
"name": field.name,
|
|
"nullable": field.nullable,
|
|
"type": _exact_blob_storage_type(field),
|
|
"metadata": {
|
|
_ARROW_EXTENSION_NAME_KEY: _BLOB_V2_EXTENSION_NAME,
|
|
},
|
|
}
|
|
value = {
|
|
"name": field.name,
|
|
"nullable": field.nullable,
|
|
"type": _exact_arrow_type(
|
|
field.type,
|
|
inside_collection=inside_collection,
|
|
collection_blobs=collection_blobs,
|
|
),
|
|
}
|
|
return value
|
|
|
|
|
|
def _exact_arrow_type(
|
|
data_type: pa.DataType,
|
|
*,
|
|
inside_collection: bool = False,
|
|
collection_blobs: bool = False,
|
|
) -> dict[str, Any]:
|
|
for candidate, name in _GRAMMAR_PRIMITIVES:
|
|
if data_type == candidate:
|
|
return {"type": name}
|
|
if pa.types.is_struct(data_type):
|
|
fields = list(data_type)
|
|
names = [field.name for field in fields]
|
|
if not fields or len(set(names)) != len(names):
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: structs must have "
|
|
"non-empty, uniquely named fields"
|
|
)
|
|
return {
|
|
"type": "struct",
|
|
"fields": [
|
|
_exact_arrow_field(
|
|
field,
|
|
inside_collection=inside_collection,
|
|
collection_blobs=collection_blobs,
|
|
)
|
|
for field in fields
|
|
],
|
|
}
|
|
if (
|
|
pa.types.is_list(data_type)
|
|
or pa.types.is_large_list(data_type)
|
|
or pa.types.is_fixed_size_list(data_type)
|
|
):
|
|
if pa.types.is_fixed_size_list(data_type):
|
|
if data_type.value_field.name != "item":
|
|
raise TypeError(
|
|
"unsupported Arrow type for Function signature: fixed-size list "
|
|
"items must be named 'item'"
|
|
)
|
|
if data_type.list_size <= 0:
|
|
raise TypeError(
|
|
f"unsupported Arrow type for Function signature: {data_type}"
|
|
)
|
|
value: dict[str, Any] = {
|
|
"type": (
|
|
"list"
|
|
if pa.types.is_list(data_type)
|
|
else "large_list"
|
|
if pa.types.is_large_list(data_type)
|
|
else "fixed_size_list"
|
|
),
|
|
"fields": [
|
|
_exact_arrow_field(
|
|
data_type.value_field,
|
|
inside_collection=True,
|
|
# Lance stores Blob v2 under list and large_list only.
|
|
collection_blobs=collection_blobs
|
|
and not pa.types.is_fixed_size_list(data_type),
|
|
)
|
|
],
|
|
}
|
|
if pa.types.is_fixed_size_list(data_type):
|
|
value["length"] = data_type.list_size
|
|
return value
|
|
if pa.types.is_map(data_type) and _data_type_has_blob_v2(data_type):
|
|
raise TypeError(_NESTED_BLOB_COLLECTION_ERROR)
|
|
raise TypeError(f"unsupported Arrow type for Function signature: {data_type}")
|
|
|
|
|
|
def _list_of(item: pa.DataType) -> pa.DataType:
|
|
return pa.list_(pa.field("item", item, nullable=False))
|
|
|
|
|
|
def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]:
|
|
nullable = False
|
|
origin = get_origin(annotation)
|
|
if origin in (Union, types.UnionType):
|
|
arguments = get_args(annotation)
|
|
non_none = tuple(
|
|
argument for argument in arguments if argument is not type(None)
|
|
)
|
|
if len(non_none) != 1 or len(non_none) == len(arguments):
|
|
raise TypeError(f"unsupported union annotation: {annotation!r}")
|
|
annotation = non_none[0]
|
|
nullable = True
|
|
|
|
origin = get_origin(annotation)
|
|
if origin is Annotated:
|
|
base, *metadata = get_args(annotation)
|
|
arrow_types = [value for value in metadata if isinstance(value, pa.DataType)]
|
|
if len(arrow_types) != 1:
|
|
raise TypeError(
|
|
"Annotated Function types require exactly one PyArrow DataType"
|
|
)
|
|
# A struct value is written as a dict, which has no Arrow type of its
|
|
# own; the Annotated metadata supplies it.
|
|
if base is dict or get_origin(base) is dict:
|
|
base_nullable = False
|
|
else:
|
|
_, base_nullable = _annotation_type(base)
|
|
return arrow_types[0], nullable or base_nullable
|
|
|
|
if isinstance(annotation, pa.DataType):
|
|
return annotation, nullable
|
|
if annotation is pa.Array or annotation is pa.RecordBatch:
|
|
raise TypeError(
|
|
f"cannot infer Arrow types from Function annotation {annotation!r}; "
|
|
"provide input_schema and output_schema together for batch callables"
|
|
)
|
|
if annotation is bool:
|
|
return pa.bool_(), nullable
|
|
if annotation is int:
|
|
return pa.int64(), nullable
|
|
if annotation is float:
|
|
return pa.float64(), nullable
|
|
if annotation is str:
|
|
return pa.string(), nullable
|
|
if annotation is bytes:
|
|
return pa.binary(), nullable
|
|
if annotation is date:
|
|
return pa.date32(), nullable
|
|
if annotation is datetime:
|
|
return pa.timestamp("us"), nullable
|
|
if get_origin(annotation) is list:
|
|
arguments = get_args(annotation)
|
|
if len(arguments) != 1:
|
|
raise TypeError(f"unsupported list annotation: {annotation!r}")
|
|
value_type, value_nullable = _annotation_type(arguments[0])
|
|
if value_nullable:
|
|
raise TypeError("nullable Function list elements are not supported")
|
|
return _list_of(value_type), nullable
|
|
raise TypeError(f"unsupported Function annotation: {annotation!r}")
|
|
|
|
|
|
def _validate_callable(function: Any) -> None:
|
|
if hasattr(function, "__wrapped__"):
|
|
raise TypeError(
|
|
"decorated wrappers are not supported; put caching inside the function "
|
|
"body or use a class Function with a cache in __init__"
|
|
)
|
|
if inspect.isgeneratorfunction(function) or inspect.isasyncgenfunction(function):
|
|
raise TypeError(
|
|
"generator Functions are not supported; return a value instead of "
|
|
"yielding, or return a list for multiple values"
|
|
)
|
|
if inspect.isfunction(function) and function.__name__ == "<lambda>":
|
|
raise TypeError("lambda Functions are not supported; use a named def instead")
|
|
|
|
|
|
def _callable_parameters(function: Callable[..., Any]) -> tuple[inspect.Parameter, ...]:
|
|
parameters = tuple(inspect.signature(function).parameters.values())
|
|
for parameter in parameters:
|
|
if parameter.kind in (
|
|
inspect.Parameter.POSITIONAL_ONLY,
|
|
inspect.Parameter.VAR_POSITIONAL,
|
|
inspect.Parameter.VAR_KEYWORD,
|
|
):
|
|
raise TypeError("Function callables require named, non-variadic parameters")
|
|
if parameter.default is not inspect.Parameter.empty:
|
|
raise TypeError(
|
|
f"Function callable defaults are not supported: {parameter.name!r}; "
|
|
"move configuration parameters to a class Function's __init__ "
|
|
"and pass them as binding-time initialization values"
|
|
)
|
|
return parameters
|
|
|
|
|
|
def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutput:
|
|
if isinstance(output, pa.Schema):
|
|
if output.metadata:
|
|
raise TypeError("Function output schema metadata is not supported")
|
|
fields = tuple(output)
|
|
elif (
|
|
isinstance(output, pa.Field)
|
|
and not _is_blob_v2_field(output)
|
|
and pa.types.is_struct(output.type)
|
|
):
|
|
_validate_exact_arrow_field(output)
|
|
if output.nullable:
|
|
raise ValueError("Function output must be non-nullable")
|
|
fields = tuple(output.type)
|
|
elif isinstance(output, pa.DataType) and pa.types.is_struct(output):
|
|
fields = tuple(output)
|
|
else:
|
|
field = (
|
|
output
|
|
if isinstance(output, pa.Field)
|
|
else pa.field("result", output, nullable=False)
|
|
)
|
|
if not isinstance(field, pa.Field):
|
|
raise TypeError(
|
|
"output_schema must be a PyArrow DataType, Field, or Schema"
|
|
)
|
|
_validate_exact_arrow_field(field)
|
|
if field.nullable and _is_blob_v2_field(field):
|
|
raise ValueError("Blob v2 Function output must be non-nullable")
|
|
return FunctionOutput(
|
|
kind="scalar",
|
|
arrow_type=_canonical_arrow_field(field, collection_blobs=True),
|
|
nullable=field.nullable,
|
|
)
|
|
|
|
if not fields:
|
|
raise ValueError("named-struct Function output must contain at least one field")
|
|
for field in fields:
|
|
_validate_exact_arrow_field(field)
|
|
names = [field.name for field in fields]
|
|
if len(set(names)) != len(names):
|
|
raise ValueError("Function output field names must be unique")
|
|
return FunctionOutput(
|
|
kind="named_struct",
|
|
fields=tuple(
|
|
FunctionResultField(
|
|
name=field.name,
|
|
arrow_type=_canonical_arrow_field(field, collection_blobs=True),
|
|
nullable=field.nullable,
|
|
)
|
|
for field in fields
|
|
),
|
|
)
|
|
|
|
|
|
def _infer_signature(
|
|
function: Callable[..., Any],
|
|
input_schema: Optional[pa.Schema],
|
|
output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
|
|
*,
|
|
method: bool = False,
|
|
) -> FunctionSignature:
|
|
parameters = _callable_parameters(function)
|
|
if method:
|
|
parameters = parameters[1:]
|
|
|
|
try:
|
|
annotations = get_type_hints(function, include_extras=True)
|
|
except Exception as error:
|
|
raise TypeError(f"failed to resolve Function annotations: {error}") from error
|
|
|
|
def input_parameter(field):
|
|
annotation = annotations.get(field.name)
|
|
if get_origin(annotation) in (Union, types.UnionType):
|
|
choices = [
|
|
value for value in get_args(annotation) if value is not type(None)
|
|
]
|
|
if len(choices) == 1:
|
|
annotation = choices[0]
|
|
arrow_type = _canonical_arrow_field(field)
|
|
if annotation is BlobReader:
|
|
if arrow_type != _FUNCTION_BLOB_V2_TYPE:
|
|
raise TypeError(
|
|
f"BlobReader input '{field.name}' requires a Blob V2 field"
|
|
)
|
|
access = "reader"
|
|
elif annotation is bytes:
|
|
if arrow_type != _FUNCTION_BLOB_V2_TYPE and not (
|
|
pa.types.is_binary(field.type) or pa.types.is_large_binary(field.type)
|
|
):
|
|
raise TypeError(
|
|
f"bytes input '{field.name}' requires binary or Blob V2"
|
|
)
|
|
access = "bytes"
|
|
else:
|
|
access = "bytes" if arrow_type == _FUNCTION_BLOB_V2_TYPE else None
|
|
return FunctionParameter(
|
|
name=field.name,
|
|
arrow_type=arrow_type,
|
|
nullable=field.nullable,
|
|
blob_access=access,
|
|
)
|
|
|
|
if input_schema is not None:
|
|
if not isinstance(input_schema, pa.Schema):
|
|
raise TypeError("input_schema must be a PyArrow Schema")
|
|
if input_schema.metadata:
|
|
raise TypeError("Function input schema metadata is not supported")
|
|
for field in input_schema:
|
|
_validate_exact_arrow_field(field)
|
|
expected = tuple(parameter.name for parameter in parameters)
|
|
actual = tuple(input_schema.names)
|
|
if actual != expected:
|
|
raise ValueError(
|
|
"input_schema fields must exactly match callable parameters in order: "
|
|
f"expected {expected!r}, got {actual!r}"
|
|
)
|
|
inputs = tuple(input_parameter(field) for field in input_schema)
|
|
else:
|
|
inputs = None
|
|
|
|
missing = []
|
|
if inputs is None:
|
|
missing += [p.name for p in parameters if p.name not in annotations]
|
|
if output_schema is None and "return" not in annotations:
|
|
missing.append("return")
|
|
if missing:
|
|
raise TypeError(f"missing Function annotations: {missing!r}")
|
|
if output_schema is not None:
|
|
output = _function_output(output_schema)
|
|
else:
|
|
output_type, output_nullable = _annotation_type(annotations["return"])
|
|
output = _function_output(
|
|
pa.field("result", output_type, nullable=output_nullable)
|
|
)
|
|
if inputs is not None:
|
|
return FunctionSignature(inputs=inputs, output=output)
|
|
|
|
inputs = []
|
|
for parameter in parameters:
|
|
annotation = annotations[parameter.name]
|
|
base = annotation
|
|
nullable = False
|
|
if get_origin(base) in (Union, types.UnionType):
|
|
choices = [value for value in get_args(base) if value is not type(None)]
|
|
if len(choices) == 1:
|
|
base, nullable = choices[0], True
|
|
if base is BlobReader:
|
|
inputs.append(
|
|
input_parameter(_blob_field(parameter.name, nullable=nullable))
|
|
)
|
|
continue
|
|
data_type, nullable = _annotation_type(annotation)
|
|
inputs.append(
|
|
input_parameter(pa.field(parameter.name, data_type, nullable=nullable))
|
|
)
|
|
return FunctionSignature(inputs=tuple(inputs), output=output)
|
|
|
|
|
|
def _is_vectorized_callable(function: Callable[..., Any], *, method: bool) -> bool:
|
|
parameters = _callable_parameters(function)
|
|
if method:
|
|
parameters = parameters[1:]
|
|
annotations = get_type_hints(function, include_extras=True)
|
|
annotated = tuple(annotations.get(parameter.name) for parameter in parameters)
|
|
return (len(annotated) == 1 and annotated[0] is pa.RecordBatch) or (
|
|
bool(annotated) and all(annotation is pa.Array for annotation in annotated)
|
|
)
|
|
|
|
|
|
# Both spellings strip from packaged source; `udf` is the deprecated alias.
|
|
_DECORATOR_NAMES = frozenset({"function", "udf"})
|
|
|
|
|
|
def _is_function_decorator(node: ast.expr) -> bool:
|
|
if isinstance(node, ast.Call):
|
|
node = node.func
|
|
return (isinstance(node, ast.Name) and node.id in _DECORATOR_NAMES) or (
|
|
isinstance(node, ast.Attribute) and node.attr in _DECORATOR_NAMES
|
|
)
|
|
|
|
|
|
def _literal_source(value: Any, name: str) -> str:
|
|
if value is None or type(value) in (bool, int, str, bytes):
|
|
return repr(value)
|
|
if type(value) is float and math.isfinite(value):
|
|
return repr(value)
|
|
if type(value) is tuple:
|
|
children = ", ".join(_literal_source(child, name) for child in value)
|
|
if len(value) == 1:
|
|
children += ","
|
|
return f"({children})"
|
|
raise TypeError(
|
|
"Function source references an unsupported global value of type "
|
|
f"{type(value).__name__} in {name!r}. Only None, bool, int, str, bytes, "
|
|
"finite float, and tuples of these literals can be captured. "
|
|
"Define this value inside the function or class body instead."
|
|
)
|
|
|
|
|
|
_DYNAMIC_NAMESPACE_ACCESS = frozenset(
|
|
{"globals", "locals", "vars", "eval", "exec", "compile", "__import__"}
|
|
)
|
|
# Modules that hand out namespaces (`sys.modules`, `builtins`, importers,
|
|
# introspection). The artifact's module namespace holds only the names it was
|
|
# packaged with, so reaching around it cannot be represented.
|
|
_NAMESPACE_MODULES = frozenset(
|
|
{"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"}
|
|
)
|
|
|
|
|
|
_NAMESPACE_HELPER_HINT = (
|
|
"Move this code into an importable helper module supplied with code=, pip=, "
|
|
"or conda=, then import and call the helper from the Function."
|
|
)
|
|
|
|
|
|
def _namespace_import_error(modules: Sequence[str]) -> ValueError:
|
|
return ValueError(
|
|
f"@function cannot package imports from namespace-access modules: {modules!r}. "
|
|
f"Imports of {', '.join(sorted(_NAMESPACE_MODULES))} (including submodules) "
|
|
"are blocked because they can access names outside the packaged "
|
|
"Function's reconstructed module namespace. " + _NAMESPACE_HELPER_HINT
|
|
)
|
|
|
|
|
|
def _namespace_imports(definition: Union[ast.FunctionDef, ast.ClassDef]) -> list[str]:
|
|
found: set[str] = set()
|
|
for node in ast.walk(definition):
|
|
if isinstance(node, ast.Import):
|
|
found.update(
|
|
alias.name
|
|
for alias in node.names
|
|
if alias.name.split(".")[0] in _NAMESPACE_MODULES
|
|
)
|
|
elif isinstance(node, ast.ImportFrom) and node.module:
|
|
if node.module.split(".")[0] in _NAMESPACE_MODULES:
|
|
found.add(node.module)
|
|
return sorted(found)
|
|
|
|
|
|
def _module_references(module_source: str) -> set[str]:
|
|
"""Names `module_source` loads from module scope, by Python's own scope
|
|
analysis on the exact text that ships: loads inside any definition, plus
|
|
what the definition statement itself evaluates at module scope (bases,
|
|
decorators, defaults). Free variables belong to an enclosing scope inside
|
|
the definition, and postponed annotations are not runtime loads."""
|
|
|
|
def visit(table: symtable.SymbolTable, found: set[str]) -> None:
|
|
for symbol in table.get_symbols():
|
|
if symbol.is_global() and (
|
|
symbol.is_referenced() or symbol.is_declared_global()
|
|
):
|
|
found.add(symbol.get_name())
|
|
for child in table.get_children():
|
|
visit(child, found)
|
|
|
|
found: set[str] = set()
|
|
module = symtable.symtable(module_source, "<function>", "exec")
|
|
for symbol in module.get_symbols():
|
|
if (
|
|
symbol.is_referenced()
|
|
and not symbol.is_assigned()
|
|
and not symbol.is_imported()
|
|
):
|
|
found.add(symbol.get_name())
|
|
for table in module.get_children():
|
|
visit(table, found)
|
|
return found
|
|
|
|
|
|
_SOURCE_HEADER = "from __future__ import annotations"
|
|
_SOURCE_ENTRY_FILE = "user_function.py"
|
|
_PYTHON_CALLABLE_ARTIFACT = "python_callable"
|
|
_PYTHON_BUNDLE_ARTIFACT = "python_bundle"
|
|
# Modules the Function image places beside user code.
|
|
_RESERVED_SOURCE_MODULES = frozenset(
|
|
{"user_function", "lance_source", "lance_callable", "lance_blob"}
|
|
)
|
|
# The service accepts bundle paths made of these components only.
|
|
_SOURCE_PATH_COMPONENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
_REQUIREMENT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")
|
|
|
|
|
|
def _class_source(cls: type) -> str:
|
|
"""Source of a module-level class.
|
|
|
|
`inspect.getsource` finds a class through its module's file, which a
|
|
notebook cell or a doctest does not have. A method's code object records
|
|
where the class body really is, so that is consulted first.
|
|
"""
|
|
for member in vars(cls).values():
|
|
if isinstance(member, (staticmethod, classmethod)):
|
|
member = member.__func__
|
|
code = (
|
|
getattr(inspect.unwrap(member), "__code__", None)
|
|
if callable(member)
|
|
else None
|
|
)
|
|
if code is None:
|
|
continue
|
|
lines = linecache.getlines(code.co_filename)
|
|
try:
|
|
module = ast.parse("".join(lines))
|
|
except SyntaxError:
|
|
continue
|
|
for node in module.body:
|
|
if (
|
|
isinstance(node, ast.ClassDef)
|
|
and node.name == cls.__name__
|
|
and node.lineno <= code.co_firstlineno <= node.end_lineno
|
|
):
|
|
start = min(
|
|
[node.lineno]
|
|
+ [decorator.lineno for decorator in node.decorator_list]
|
|
)
|
|
return "".join(lines[start - 1 : node.end_lineno])
|
|
return inspect.getsource(cls)
|
|
|
|
|
|
def _definition_node(
|
|
target: Any, *, entry: bool
|
|
) -> Union[ast.FunctionDef, ast.ClassDef]:
|
|
try:
|
|
source = textwrap.dedent(
|
|
_class_source(target)
|
|
if inspect.isclass(target)
|
|
else inspect.getsource(target)
|
|
)
|
|
except (OSError, TypeError) as error:
|
|
raise ValueError(
|
|
f"@function requires inspectable Python source for {target.__qualname__!r}"
|
|
) from error
|
|
module = ast.parse(source)
|
|
is_class = inspect.isclass(target)
|
|
definitions = [
|
|
node
|
|
for node in module.body
|
|
if isinstance(
|
|
node,
|
|
(ast.ClassDef,) if is_class else (ast.FunctionDef, ast.AsyncFunctionDef),
|
|
)
|
|
and node.name == target.__name__
|
|
]
|
|
if len(definitions) != 1 or isinstance(definitions[0], ast.AsyncFunctionDef):
|
|
raise ValueError(
|
|
"@function source must contain exactly one "
|
|
+ ("class" if is_class else "synchronous function")
|
|
)
|
|
definition = definitions[0]
|
|
if entry:
|
|
if any(
|
|
not _is_function_decorator(decorator)
|
|
for decorator in definition.decorator_list
|
|
):
|
|
raise ValueError("@function cannot package additional Python decorators")
|
|
definition.decorator_list = []
|
|
return definition
|
|
|
|
|
|
def _definition_namespace(target: Any) -> Mapping[str, Any]:
|
|
"""The module namespace a function or class resolves its globals in."""
|
|
if inspect.isclass(target):
|
|
module = sys.modules.get(target.__module__)
|
|
if module is None:
|
|
raise ValueError(
|
|
f"@function cannot resolve the module of {target.__qualname__!r}"
|
|
)
|
|
namespace = vars(module)
|
|
environment = namespace.get("__builtins__", builtins)
|
|
environment = (
|
|
vars(environment)
|
|
if isinstance(environment, types.ModuleType)
|
|
else environment
|
|
)
|
|
else:
|
|
namespace = target.__globals__
|
|
environment = target.__builtins__
|
|
# The artifact runs under the standard builtins; only the exact mapping is
|
|
# provably equivalent (a subclass or copy can change lookups and hooks).
|
|
if environment is not vars(builtins):
|
|
raise ValueError("@function cannot package a non-standard builtins environment")
|
|
return namespace
|
|
|
|
|
|
def _is_self_reference(target: Any, namespace: Mapping[str, Any], name: str) -> bool:
|
|
"""`name` inside the definition means the definition itself unless the
|
|
module has since bound it to something else."""
|
|
if name != target.__name__:
|
|
return False
|
|
bound = namespace.get(name, target)
|
|
if bound is target:
|
|
return True
|
|
# The decorator's own result is the one wrapper known to behave like
|
|
# `target`; any other binding may differ from the packaged definition.
|
|
return type(bound) is FunctionDefinition and bound._function is target
|
|
|
|
|
|
def _packaged_by_source(value: Any) -> bool:
|
|
"""Functions and classes defined in `__main__` -- notebook cells and
|
|
scripts -- have no module a worker can import, so their source travels."""
|
|
if not (inspect.isfunction(value) or inspect.isclass(value)):
|
|
return False
|
|
if getattr(value, "__module__", None) != "__main__":
|
|
return False
|
|
qualname = value.__qualname__
|
|
if "<" in qualname or "." in qualname:
|
|
raise ValueError(
|
|
f"@function cannot package {qualname!r}: only module-level functions and "
|
|
"classes travel with a Function, not lambdas, closures, or nested "
|
|
"definitions"
|
|
)
|
|
return True
|
|
|
|
|
|
def _validate_package_requirements(kind: str, packages: Sequence[str]) -> None:
|
|
for index, package in enumerate(packages):
|
|
field = f"{kind}[{index}]"
|
|
if not isinstance(package, str):
|
|
raise TypeError(f"{field} must be a string, got {package!r}")
|
|
if not package or package != package.strip():
|
|
raise ValueError(
|
|
f"{field} requirement {package!r} must be non-empty and trimmed"
|
|
)
|
|
try:
|
|
if kind == "conda":
|
|
_validate_conda_requirement(package)
|
|
else:
|
|
candidate = package
|
|
if re.match(r"^[A-Za-z][A-Za-z0-9+.-]*://", package):
|
|
candidate = f"dependency @ {package}"
|
|
Requirement(candidate)
|
|
except ValueError as error:
|
|
raise ValueError(
|
|
f"invalid {field} requirement {package!r}: {error}"
|
|
) from error
|
|
|
|
|
|
def _requirement_name(requirement: str) -> str:
|
|
requirement = requirement.rpartition("::")[2]
|
|
match = _REQUIREMENT_NAME.match(requirement.strip())
|
|
return _normalized_distribution(match.group(0) if match else requirement)
|
|
|
|
|
|
def _normalized_distribution(name: str) -> str:
|
|
return re.sub(r"[-_.]+", "_", name).lower()
|
|
|
|
|
|
def _installed_roots() -> tuple[str, ...]:
|
|
import os
|
|
import site
|
|
import sysconfig
|
|
|
|
roots = set(site.getsitepackages())
|
|
roots.add(site.getusersitepackages())
|
|
paths = sysconfig.get_paths()
|
|
for key in ("purelib", "platlib", "stdlib", "platstdlib"):
|
|
if key in paths:
|
|
roots.add(paths[key])
|
|
return tuple(os.path.realpath(root) + os.sep for root in roots)
|
|
|
|
|
|
def _module_locations(module: types.ModuleType) -> tuple[str, ...]:
|
|
import os
|
|
|
|
file = getattr(module, "__file__", None)
|
|
if file:
|
|
return (os.path.realpath(file),)
|
|
return tuple(os.path.realpath(path) for path in getattr(module, "__path__", ()))
|
|
|
|
|
|
def _module_locations_for_name(module_name: str) -> tuple[str, ...]:
|
|
"""Where a top-level module would be imported from, without importing it
|
|
when it is not already loaded."""
|
|
import os
|
|
|
|
top = module_name.partition(".")[0]
|
|
module = sys.modules.get(top)
|
|
if module is not None:
|
|
return _module_locations(module)
|
|
try:
|
|
spec = importlib.util.find_spec(top)
|
|
except (AttributeError, ImportError, ValueError):
|
|
return ()
|
|
if spec is None:
|
|
return ()
|
|
if spec.submodule_search_locations:
|
|
return tuple(os.path.realpath(path) for path in spec.submodule_search_locations)
|
|
if spec.origin and spec.origin not in {"built-in", "frozen"}:
|
|
return (os.path.realpath(spec.origin),)
|
|
return ()
|
|
|
|
|
|
class _SourcePackager:
|
|
"""Builds the entry module of a Function artifact.
|
|
|
|
The module holds the definition plus exactly the module-level names it
|
|
references: modules as imports, importable classes and functions as
|
|
imports, literals inline, and functions and classes defined in
|
|
``__main__`` by source, recursively, dependencies first.
|
|
"""
|
|
|
|
def __init__(self, code_modules: frozenset[str], requirements: frozenset[str]):
|
|
self._code_modules = code_modules
|
|
self._requirements = requirements
|
|
self._imports: dict[str, str] = {}
|
|
self._definitions: list[str] = []
|
|
self._packaged: set[int] = set()
|
|
self._names: dict[str, Any] = {}
|
|
self._warned_imports: set[str] = set()
|
|
|
|
def package(self, target: Any, signature: FunctionSignature) -> str:
|
|
entry = self._definition(target, entry=True, signature=signature)
|
|
parts = [_SOURCE_HEADER]
|
|
if self._imports:
|
|
parts.extend(["", *(self._imports[name] for name in sorted(self._imports))])
|
|
for definition in self._definitions:
|
|
parts.extend(["", definition])
|
|
parts.extend(["", entry, ""])
|
|
return "\n".join(parts)
|
|
|
|
def _definition(
|
|
self, target: Any, *, entry: bool, signature: Optional[FunctionSignature] = None
|
|
) -> str:
|
|
definition = _definition_node(target, entry=entry)
|
|
if not inspect.isclass(target) and inspect.getclosurevars(target).nonlocals:
|
|
raise ValueError(
|
|
"@function cannot package functions that capture closure values"
|
|
)
|
|
namespace = _definition_namespace(target)
|
|
reader_name = (
|
|
self._blob_annotations(definition, signature, namespace)
|
|
if signature is not None
|
|
else None
|
|
)
|
|
source = ast.unparse(definition)
|
|
references = _module_references(f"{_SOURCE_HEADER}\n\n{source}\n")
|
|
dynamic = sorted(references & _DYNAMIC_NAMESPACE_ACCESS)
|
|
if dynamic:
|
|
raise ValueError(
|
|
f"@function cannot package dynamic namespace access: {dynamic!r}. "
|
|
"These operations can access names outside the packaged "
|
|
"Function's reconstructed module namespace. Reference dependencies "
|
|
"explicitly. " + _NAMESPACE_HELPER_HINT
|
|
)
|
|
namespace_imports = _namespace_imports(definition)
|
|
if namespace_imports:
|
|
raise _namespace_import_error(namespace_imports)
|
|
self._warn_unshipped_direct_imports(definition)
|
|
self._packaged.add(id(target))
|
|
self._claim(target.__name__, ("definition", id(target)))
|
|
if reader_name is not None:
|
|
self._bind(reader_name, BlobReader)
|
|
unresolved = []
|
|
for name in sorted(references):
|
|
if name == target.__name__:
|
|
if not _is_self_reference(target, namespace, name):
|
|
raise ValueError(
|
|
f"@function cannot package {name!r}: the module binds that "
|
|
"name to another value, which the artifact's own "
|
|
"definition would shadow"
|
|
)
|
|
continue
|
|
if name in namespace:
|
|
self._bind(name, namespace[name])
|
|
elif not hasattr(builtins, name):
|
|
unresolved.append(name)
|
|
if unresolved:
|
|
raise ValueError(
|
|
f"@function source contains unresolved global names: {unresolved!r}"
|
|
)
|
|
return source
|
|
|
|
def _blob_annotations(
|
|
self,
|
|
definition: Union[ast.FunctionDef, ast.ClassDef],
|
|
signature: FunctionSignature,
|
|
namespace: Mapping[str, Any],
|
|
) -> Optional[str]:
|
|
"""Bind reader inputs to the worker type, including postponed aliases."""
|
|
readers = {
|
|
parameter.name: parameter
|
|
for parameter in signature.inputs
|
|
if parameter.blob_access == "reader"
|
|
}
|
|
if not readers:
|
|
return None
|
|
call = definition
|
|
if isinstance(definition, ast.ClassDef):
|
|
call = next(
|
|
(
|
|
member
|
|
for member in definition.body
|
|
if isinstance(member, ast.FunctionDef) and member.name == "__call__"
|
|
),
|
|
None,
|
|
)
|
|
if call is None:
|
|
return None
|
|
# Avoid shadowing globals used by the body or other packaged definitions.
|
|
name = "_lance_blob_reader"
|
|
while name in namespace or name in self._names or name == definition.name:
|
|
name += "_"
|
|
for argument in (*call.args.args, *call.args.kwonlyargs):
|
|
if argument.arg in readers:
|
|
annotation = name
|
|
if readers[argument.arg].nullable:
|
|
annotation += " | None"
|
|
argument.annotation = ast.parse(annotation, mode="eval").body
|
|
return name
|
|
|
|
def _claim(self, name: str, meaning: Any) -> None:
|
|
claimed = self._names.setdefault(name, meaning)
|
|
if claimed != meaning:
|
|
raise ValueError(
|
|
f"@function cannot package {name!r}: packaged definitions bind that "
|
|
"name to different values"
|
|
)
|
|
|
|
def _bind(self, name: str, value: Any) -> None:
|
|
if _packaged_by_source(value):
|
|
if value.__name__ != name:
|
|
raise ValueError(
|
|
f"@function packages {value.__qualname__!r} by source, so it "
|
|
f"must be referenced by its own name, not {name!r}"
|
|
)
|
|
if id(value) not in self._packaged:
|
|
self._definitions.append(self._definition(value, entry=False))
|
|
self._claim(name, ("definition", id(value)))
|
|
return
|
|
line = self._global_source(name, value)
|
|
self._claim(name, line)
|
|
self._imports[name] = line
|
|
|
|
def _global_source(self, name: str, value: Any) -> str:
|
|
"""One module-level line that rebinds `name` to `value` in the artifact:
|
|
an import for modules and importable classes/functions, a literal
|
|
otherwise."""
|
|
if value is BlobReader:
|
|
return f"from lance_blob import BlobReader as {name}"
|
|
if isinstance(value, types.ModuleType):
|
|
if value.__name__.split(".")[0] in _NAMESPACE_MODULES:
|
|
raise _namespace_import_error([value.__name__])
|
|
try:
|
|
imported = importlib.import_module(value.__name__)
|
|
except ImportError:
|
|
imported = None
|
|
if imported is not value:
|
|
raise TypeError(
|
|
f"Function source references module {name!r} that does not "
|
|
f"import as {value.__name__!r}"
|
|
)
|
|
self._require_importable(value.__name__)
|
|
return f"import {value.__name__} as {name}"
|
|
module_name = getattr(value, "__module__", None)
|
|
qualname = getattr(value, "__qualname__", None)
|
|
if (
|
|
isinstance(module_name, str)
|
|
and isinstance(qualname, str)
|
|
and module_name != "__main__"
|
|
and "." not in qualname
|
|
and "<" not in qualname
|
|
):
|
|
try:
|
|
imported = getattr(importlib.import_module(module_name), qualname)
|
|
except (ImportError, AttributeError):
|
|
imported = None
|
|
if imported is value:
|
|
self._require_importable(module_name)
|
|
return f"from {module_name} import {qualname} as {name}"
|
|
return f"{name} = {_literal_source(value, name)}"
|
|
|
|
def _require_importable(self, module_name: str) -> None:
|
|
"""Warn about an import the Function's environment may not be able to
|
|
satisfy.
|
|
|
|
An unresolved module warns only when the Function declares no package
|
|
dependencies. A local source module warns unless ``code=`` ships it or
|
|
a declared requirement matches its top-level name.
|
|
|
|
A warning rather than an error: a distribution name need not match
|
|
the module it installs, so a declared package can still provide the
|
|
module, and callers may rewrite the packaged source before
|
|
registering it."""
|
|
top = module_name.partition(".")[0]
|
|
if (
|
|
top in self._code_modules
|
|
or top in sys.stdlib_module_names
|
|
or top in sys.builtin_module_names
|
|
or _normalized_distribution(top) in self._requirements
|
|
or top in self._warned_imports
|
|
):
|
|
return
|
|
locations = _module_locations_for_name(top)
|
|
if not locations:
|
|
if not self._requirements:
|
|
self._warned_imports.add(top)
|
|
warnings.warn(
|
|
f"@function source imports {top!r}, which is unavailable in the "
|
|
"client environment and has no declared pip/conda dependencies. "
|
|
"Remote availability cannot be verified; declare the required "
|
|
"package or confirm that the base image provides it.",
|
|
UserWarning,
|
|
)
|
|
return
|
|
roots = _installed_roots()
|
|
if any(location.startswith(roots) for location in locations):
|
|
return
|
|
self._warned_imports.add(top)
|
|
warnings.warn(
|
|
f"@function source imports {module_name!r} from {locations[0]}, a local "
|
|
"module the Function's environment may not be able to import; ship "
|
|
f"it with code=[{top}] or declare the package that provides it in "
|
|
"pip/conda",
|
|
UserWarning,
|
|
)
|
|
|
|
def _warn_unshipped_direct_imports(
|
|
self, definition: Union[ast.FunctionDef, ast.ClassDef]
|
|
) -> None:
|
|
"""Warn for body imports that resolve to local, undeclared source.
|
|
|
|
Imports from site-packages are deliberately not guessed about: the
|
|
declared distribution name need not match the imported module, and the
|
|
client cannot inspect the remote base image.
|
|
"""
|
|
for node in ast.walk(definition):
|
|
if isinstance(node, ast.Import):
|
|
for alias in node.names:
|
|
self._require_importable(alias.name)
|
|
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
|
|
self._require_importable(node.module)
|
|
|
|
|
|
def _code_files(
|
|
modules: Sequence[Any],
|
|
) -> tuple[dict[str, str], dict[str, str], frozenset[str]]:
|
|
"""Source and byte-preserving package resources on the Function import path."""
|
|
files: dict[str, str] = {}
|
|
resources: dict[str, str] = {}
|
|
names: set[str] = set()
|
|
for module in modules:
|
|
if not isinstance(module, types.ModuleType):
|
|
raise TypeError(
|
|
f"code= takes imported modules or packages, got {type(module).__name__}"
|
|
)
|
|
name = module.__name__
|
|
if "." in name:
|
|
raise ValueError(
|
|
f"code= takes top-level modules or packages; pass "
|
|
f"{name.partition('.')[0]!r} instead of {name!r}"
|
|
)
|
|
if (
|
|
name in _RESERVED_SOURCE_MODULES
|
|
or name == "__main__"
|
|
or name in sys.stdlib_module_names
|
|
or not _SOURCE_PATH_COMPONENT.fullmatch(name)
|
|
):
|
|
raise ValueError(f"code= cannot ship module {name!r}")
|
|
if name in names:
|
|
raise ValueError(f"code= lists module {name!r} more than once")
|
|
names.add(name)
|
|
paths = list(getattr(module, "__path__", ()))
|
|
if paths:
|
|
if len(paths) != 1 or not getattr(module, "__file__", None):
|
|
raise ValueError(
|
|
f"code= cannot ship namespace package {name!r}; give it an "
|
|
"__init__.py"
|
|
)
|
|
root = Path(paths[0])
|
|
sources = []
|
|
skipped = set()
|
|
for directory, subdirectories, filenames in os.walk(root):
|
|
kept = []
|
|
for child in sorted(subdirectories):
|
|
path = Path(directory) / child
|
|
if child == "__pycache__" or child.startswith("."):
|
|
continue
|
|
if not path.is_symlink() and re.fullmatch(
|
|
r"[A-Za-z0-9_][A-Za-z0-9_.-]*", child
|
|
):
|
|
kept.append(child)
|
|
else:
|
|
relative = path.relative_to(root)
|
|
skipped.add(f"{name}/{relative.as_posix()}/")
|
|
subdirectories[:] = kept
|
|
for filename in sorted(filenames):
|
|
path = Path(directory) / filename
|
|
stem, suffix = os.path.splitext(filename)
|
|
if suffix in {".pyc", ".pyo"} or filename.startswith("."):
|
|
continue
|
|
relative = path.relative_to(root)
|
|
key = f"{name}/{relative.as_posix()}"
|
|
if (
|
|
path.is_symlink()
|
|
or not path.is_file()
|
|
or not re.fullmatch(r"[A-Za-z0-9_][A-Za-z0-9_.-]*", filename)
|
|
):
|
|
skipped.add(key)
|
|
elif suffix == ".py":
|
|
if all(
|
|
_SOURCE_PATH_COMPONENT.fullmatch(part)
|
|
for part in relative.with_suffix("").parts
|
|
):
|
|
sources.append(path)
|
|
else:
|
|
skipped.add(key)
|
|
else:
|
|
resources[key] = base64.b64encode(path.read_bytes()).decode(
|
|
"ascii"
|
|
)
|
|
if skipped:
|
|
warnings.warn(
|
|
"code= cannot ship these package paths: "
|
|
f"{', '.join(sorted(skipped))}. "
|
|
"Use regular files with safe relative names and importable "
|
|
"Python module names.",
|
|
UserWarning,
|
|
)
|
|
for path in sources:
|
|
relative = path.relative_to(root).as_posix()
|
|
files[f"{name}/{relative}"] = _source_text(path)
|
|
else:
|
|
file = getattr(module, "__file__", None)
|
|
if not file or not file.endswith(".py"):
|
|
raise ValueError(f"code= ships Python source; module {name!r} has none")
|
|
files[f"{name}.py"] = _source_text(Path(file))
|
|
return files, resources, frozenset(names)
|
|
|
|
|
|
def _source_text(path) -> str:
|
|
try:
|
|
return path.read_bytes().decode("utf-8")
|
|
except UnicodeDecodeError as error:
|
|
raise ValueError(f"code= ships UTF-8 source; {path} is not UTF-8") from error
|
|
|
|
|
|
def _initialization_type(data_type: pa.DataType) -> bool:
|
|
"""Types with one unambiguous JSON and SQL literal spelling."""
|
|
if data_type in (
|
|
pa.bool_(),
|
|
pa.int8(),
|
|
pa.int16(),
|
|
pa.int32(),
|
|
pa.int64(),
|
|
pa.uint8(),
|
|
pa.uint16(),
|
|
pa.uint32(),
|
|
pa.uint64(),
|
|
pa.float32(),
|
|
pa.float64(),
|
|
pa.string(),
|
|
pa.large_string(),
|
|
):
|
|
return True
|
|
if (
|
|
pa.types.is_list(data_type)
|
|
or pa.types.is_large_list(data_type)
|
|
or pa.types.is_fixed_size_list(data_type)
|
|
):
|
|
return _initialization_type(data_type.value_type)
|
|
if pa.types.is_struct(data_type):
|
|
return all(_initialization_type(field.type) for field in data_type)
|
|
return False
|
|
|
|
|
|
def _initialization_signature(cls: type) -> tuple[FunctionParameter, ...]:
|
|
"""The initialization fields a class Function's ``__init__`` declares."""
|
|
if cls.__init__ is object.__init__:
|
|
return ()
|
|
parameters = tuple(inspect.signature(cls.__init__).parameters.values())[1:]
|
|
for parameter in parameters:
|
|
if parameter.kind not in (
|
|
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
inspect.Parameter.KEYWORD_ONLY,
|
|
):
|
|
raise TypeError(
|
|
"Function initialization parameters must be named and "
|
|
f"non-variadic: {parameter.name!r}"
|
|
)
|
|
try:
|
|
annotations = get_type_hints(cls.__init__, include_extras=True)
|
|
except Exception as error:
|
|
raise TypeError(
|
|
f"failed to resolve Function initialization annotations: {error}"
|
|
) from error
|
|
missing = [
|
|
parameter.name for parameter in parameters if parameter.name not in annotations
|
|
]
|
|
if missing:
|
|
raise TypeError(f"missing Function initialization annotations: {missing!r}")
|
|
fields = []
|
|
for parameter in parameters:
|
|
data_type, nullable = _annotation_type(annotations[parameter.name])
|
|
if not _initialization_type(data_type):
|
|
raise TypeError(
|
|
f"Function initialization parameter {parameter.name!r} has type "
|
|
f"{data_type}; initialization takes booleans, integers, floats, "
|
|
"strings, and lists or structs of them"
|
|
)
|
|
# A parameter with a default may be omitted from a binding; a null
|
|
# value then takes the default.
|
|
if parameter.default is not inspect.Parameter.empty:
|
|
nullable = True
|
|
fields.append(
|
|
FunctionParameter(
|
|
name=parameter.name,
|
|
arrow_type=_canonical_arrow_field(
|
|
pa.field(parameter.name, data_type, nullable=nullable)
|
|
),
|
|
nullable=nullable,
|
|
)
|
|
)
|
|
return tuple(fields)
|
|
|
|
|
|
def _class_member(cls: type, name: str) -> Any:
|
|
for klass in cls.__mro__:
|
|
if klass is not object and name in vars(klass):
|
|
return vars(klass)[name]
|
|
return None
|
|
|
|
|
|
def _class_call(cls: type) -> Callable[..., Any]:
|
|
call = _class_member(cls, "__call__")
|
|
_validate_callable(call)
|
|
if not inspect.isfunction(call) or inspect.iscoroutinefunction(call):
|
|
raise TypeError("a class Function must define a synchronous __call__ method")
|
|
close = _class_member(cls, "close")
|
|
if close is not None and (
|
|
not inspect.isfunction(close) or len(inspect.signature(close).parameters) != 1
|
|
):
|
|
raise TypeError("a class Function's close must be a method taking no arguments")
|
|
return call
|
|
|
|
|
|
class FunctionDefinition:
|
|
"""A Python callable prepared for remote Function registration.
|
|
|
|
Instances are created with :func:`function`. Calling an instance is ordinary
|
|
local Python: a decorated function is called directly, while a decorated
|
|
class is constructed using its initialization fields and the returned
|
|
instance is called with each input. Reuse the instance and call its
|
|
``close`` method when it defines one. This public call does not create a
|
|
remote application or build the declared remote runtime, and it does not
|
|
need the private implementation attribute.
|
|
|
|
Remote execution adapts the callable to the internal Arrow batch ABI
|
|
described by the registration artifact. By default it invokes the callable
|
|
once per row. A callable whose parameters are all annotated as
|
|
``pyarrow.Array`` receives whole Arrow arrays and returns a
|
|
``pyarrow.Array`` instead. This authoring choice is independent of the
|
|
internal artifact adapter in the registration request. GPU Functions should
|
|
use the Arrow form when the model supports batches: a scalar GPU callable is
|
|
invoked once per row and leaves the model's own batching unavailable.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
function: Any,
|
|
*,
|
|
name: Optional[str],
|
|
input_schema: Optional[pa.Schema],
|
|
output_schema: Optional[pa.DataType | pa.Field | pa.Schema],
|
|
pip: tuple[str, ...],
|
|
env: Mapping[str, str],
|
|
python_version: Optional[str],
|
|
batch_size: Optional[int] = None,
|
|
max_concurrency: Optional[int] = None,
|
|
row_timeout_seconds: Optional[float] = None,
|
|
retry_policy: Optional[FunctionRetryPolicy] = None,
|
|
gpu: bool = False,
|
|
gpu_class: Optional[str] = None,
|
|
conda: tuple[str, ...] = (),
|
|
conda_channels: tuple[str, ...] = (),
|
|
apt: tuple[str, ...] = (),
|
|
image: Optional[str] = None,
|
|
code: tuple[types.ModuleType, ...] = (),
|
|
models: tuple[HuggingFaceModel, ...] = (),
|
|
):
|
|
is_class = inspect.isclass(function)
|
|
if not is_class and (
|
|
not inspect.isfunction(function) or inspect.iscoroutinefunction(function)
|
|
):
|
|
_validate_callable(function)
|
|
raise TypeError(
|
|
"@function requires a synchronous Python function or a class"
|
|
)
|
|
if pip and conda:
|
|
raise ValueError("a Function environment is pip or conda, not both")
|
|
if conda_channels and not conda:
|
|
raise ValueError("conda_channels requires conda packages")
|
|
_validate_package_requirements("conda" if conda else "pip", conda or pip)
|
|
gpu_marker = _normalize_gpu_marker(gpu)
|
|
if gpu_class is not None:
|
|
_validate_gpu_class_name(gpu_class)
|
|
if gpu_marker is None:
|
|
raise ValueError("gpu_class requires gpu=True")
|
|
packages = tuple(sorted(set(conda if conda else pip)))
|
|
system_packages = tuple(sorted(set(apt)))
|
|
if any(
|
|
not package or package != package.strip()
|
|
for package in packages + system_packages
|
|
):
|
|
raise ValueError("package requirements must be non-empty and trimmed")
|
|
if image is not None and not _IMAGE_REFERENCE.fullmatch(image):
|
|
raise ValueError(
|
|
f"Function image {image!r} must be host/repository@sha256:<digest>"
|
|
)
|
|
if any(not isinstance(model, HuggingFaceModel) for model in models):
|
|
raise TypeError("models must contain HuggingFaceModel values")
|
|
repos = [model.repo for model in models]
|
|
duplicates = sorted({repo for repo in repos if repos.count(repo) > 1})
|
|
if duplicates:
|
|
raise ValueError(
|
|
"a Function declares each Hugging Face repository once; "
|
|
f"duplicated: {duplicates!r}"
|
|
)
|
|
if conda:
|
|
environment_spec = PythonEnvironmentSpec(
|
|
kind="conda",
|
|
packages=packages,
|
|
channels=tuple(conda_channels),
|
|
image=image,
|
|
system_packages=system_packages,
|
|
)
|
|
else:
|
|
environment_spec = PythonEnvironmentSpec(
|
|
kind="pip",
|
|
packages=packages,
|
|
image=image,
|
|
system_packages=system_packages,
|
|
)
|
|
environment = dict(env)
|
|
if any(
|
|
not isinstance(key, str) or not isinstance(value, str)
|
|
for key, value in environment.items()
|
|
):
|
|
raise TypeError("Function env keys and values must be strings")
|
|
reserved = sorted(
|
|
set(environment).intersection(
|
|
{"HF_HOME", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"}
|
|
)
|
|
)
|
|
if models and reserved:
|
|
raise ValueError(
|
|
"model Functions reserve environment variables "
|
|
f"{reserved!r}; the worker sets them for the private snapshot"
|
|
)
|
|
if not is_class:
|
|
_validate_callable(function)
|
|
function_name = name or function.__name__
|
|
if not _FUNCTION_NAME.fullmatch(function_name):
|
|
raise ValueError(f"invalid Function name: {function_name!r}")
|
|
if is_class:
|
|
if "<" in function.__qualname__ or "." in function.__qualname__:
|
|
raise ValueError(
|
|
"@function classes must be defined at module level; nested classes "
|
|
"would capture their enclosing scope"
|
|
)
|
|
signature = _infer_signature(
|
|
_class_call(function), input_schema, output_schema, method=True
|
|
)
|
|
initialization = _initialization_signature(function)
|
|
shared = sorted(
|
|
{field.name for field in initialization}
|
|
& {parameter.name for parameter in signature.inputs}
|
|
)
|
|
if shared:
|
|
raise ValueError(
|
|
"Function initialization parameters and inputs share names "
|
|
f"{shared!r}; a binding passes both as keyword arguments"
|
|
)
|
|
signature = signature._copy(update={"initialization": initialization})
|
|
else:
|
|
signature = _infer_signature(function, input_schema, output_schema)
|
|
if gpu_marker is not None:
|
|
call = _class_call(function) if is_class else function
|
|
if not _is_vectorized_callable(call, method=is_class):
|
|
warnings.warn(
|
|
f"GPU Function {function_name!r} uses a scalar callable, so each "
|
|
"row invokes it separately and its model batch is not inferred. "
|
|
"For GPU throughput, annotate every input as pyarrow.Array or "
|
|
"take one pyarrow.RecordBatch, provide input_schema and "
|
|
"output_schema, and batch inside that callable.",
|
|
UserWarning,
|
|
stacklevel=3,
|
|
)
|
|
code_files, code_resources, code_modules = _code_files(code)
|
|
entry_source = _SourcePackager(
|
|
code_modules, frozenset(_requirement_name(package) for package in packages)
|
|
).package(function, signature)
|
|
if code_files:
|
|
kind = _PYTHON_BUNDLE_ARTIFACT
|
|
bundle = {"files": {**code_files, _SOURCE_ENTRY_FILE: entry_source}}
|
|
if code_resources:
|
|
bundle["resources"] = code_resources
|
|
source = json.dumps(
|
|
bundle,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
else:
|
|
kind = _PYTHON_CALLABLE_ARTIFACT
|
|
source = entry_source.encode("utf-8")
|
|
digest = f"sha256:{hashlib.sha256(source).hexdigest()}"
|
|
runtime = PythonRuntimeSpec(
|
|
kind="python_v2" if gpu_marker is not None else "python",
|
|
python_version=python_version
|
|
or f"{sys.version_info.major}.{sys.version_info.minor}",
|
|
environment=environment_spec,
|
|
env=environment,
|
|
gpu=gpu_marker,
|
|
gpu_class=gpu_class,
|
|
)
|
|
self._function = function
|
|
self._request = FunctionRegistrationRequest(
|
|
name=function_name,
|
|
artifact=FunctionArtifactRequest(
|
|
kind=kind,
|
|
digest=digest,
|
|
entrypoint=function.__name__,
|
|
content=FunctionArtifactContent(
|
|
encoding="base64",
|
|
data=base64.b64encode(source).decode("ascii"),
|
|
),
|
|
adapter=PythonAdapterSpec(
|
|
kind="scalar_to_arrow_batch",
|
|
version=1,
|
|
),
|
|
),
|
|
signature=signature,
|
|
runtime=runtime,
|
|
batch_size=batch_size,
|
|
max_concurrency=max_concurrency,
|
|
row_timeout_seconds=row_timeout_seconds,
|
|
retry_policy=retry_policy,
|
|
models=models,
|
|
)
|
|
# A class's own __dict__ holds its methods; copying it would shadow
|
|
# this wrapper's attributes.
|
|
functools.update_wrapper(
|
|
self, function, updated=() if is_class else functools.WRAPPER_UPDATES
|
|
)
|
|
|
|
@property
|
|
def registration_request(self) -> FunctionRegistrationRequest:
|
|
"""The immutable request sent by ``create_function_async``.
|
|
|
|
Carries no secret bindings. Binding is a registration-time decision,
|
|
so a Function bound to Secrets is registered through :meth:`bind_secrets`,
|
|
which is what ``create_function`` calls.
|
|
"""
|
|
return self._request
|
|
|
|
def bind_secrets(
|
|
self, secrets: Optional[Sequence[EnvVarSecret]]
|
|
) -> FunctionRegistrationRequest:
|
|
"""The registration request for this definition bound to ``secrets``.
|
|
|
|
Binding does not change the Function's source: each
|
|
[EnvVarSecret][lancedb.secrets.EnvVarSecret] names a Secret and the
|
|
environment variable its value should arrive in, and the Function reads
|
|
that variable the way it already did. Whether the named Secrets exist is
|
|
the server's answer, not this one.
|
|
"""
|
|
bindings = () if secrets is None else tuple(secrets)
|
|
wrong_type = [
|
|
binding for binding in bindings if not isinstance(binding, EnvVarSecret)
|
|
]
|
|
if wrong_type:
|
|
kinds = sorted({type(binding).__name__ for binding in wrong_type})
|
|
raise TypeError(
|
|
f"Function secrets must be EnvVarSecret values, not {kinds!r}; a "
|
|
"credential value is never sent to this API"
|
|
)
|
|
variables = [binding.env_variable for binding in bindings]
|
|
duplicates = sorted({name for name in variables if variables.count(name) > 1})
|
|
if duplicates:
|
|
raise ValueError(
|
|
"a Function binds each environment variable once; duplicated: "
|
|
f"{duplicates!r}"
|
|
)
|
|
# `env` is ordinary configuration carried in the definition, so a name in
|
|
# both would have a value visible in the Function's record and a value
|
|
# that is not. Refuse rather than pick.
|
|
environment = self._request.runtime.env or {}
|
|
overlap = sorted(set(environment) & set(variables))
|
|
if overlap:
|
|
raise ValueError(
|
|
f"Function env and secret bindings must be disjoint: {overlap!r}"
|
|
)
|
|
if not bindings:
|
|
return self._request
|
|
# Sorted, because the list is carried in the FunctionVersion hash and a
|
|
# caller's argument order is not part of what a Function is.
|
|
resolved = tuple(
|
|
sorted(
|
|
(
|
|
SecretBinding(
|
|
kind="env",
|
|
variable=binding.env_variable,
|
|
secret_ref=SecretReference(
|
|
name=binding.secret_name,
|
|
namespace_path=tuple(binding.secret_namespace_path),
|
|
),
|
|
)
|
|
for binding in bindings
|
|
),
|
|
key=lambda binding: (binding.kind, binding.variable or ""),
|
|
)
|
|
)
|
|
return self._request._copy(update={"secret_bindings": resolved})
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
return self._function(*args, **kwargs)
|
|
|
|
|
|
@overload
|
|
def function(target: Callable[..., Any]) -> FunctionDefinition: ...
|
|
|
|
|
|
@overload
|
|
def function(
|
|
target: None = None,
|
|
*,
|
|
name: Optional[str] = None,
|
|
input_schema: Optional[pa.Schema] = None,
|
|
output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
|
|
pip: tuple[str, ...] | list[str] = (),
|
|
env: Optional[Mapping[str, str]] = None,
|
|
python_version: Optional[str] = None,
|
|
batch_size: Optional[int] = None,
|
|
max_concurrency: Optional[int] = None,
|
|
row_timeout_seconds: Optional[float] = None,
|
|
retry_policy: Optional[FunctionRetryPolicy] = None,
|
|
gpu: bool = False,
|
|
gpu_class: Optional[str] = None,
|
|
conda: tuple[str, ...] | list[str] = (),
|
|
conda_channels: tuple[str, ...] | list[str] = (),
|
|
apt: tuple[str, ...] | list[str] = (),
|
|
image: Optional[str] = None,
|
|
code: Sequence[types.ModuleType] = (),
|
|
models: Sequence[HuggingFaceModel] = (),
|
|
) -> Callable[[Callable[..., Any]], FunctionDefinition]: ...
|
|
|
|
|
|
def function(
|
|
target: Optional[Callable[..., Any]] = None,
|
|
*,
|
|
name: Optional[str] = None,
|
|
input_schema: Optional[pa.Schema] = None,
|
|
output_schema: Optional[pa.DataType | pa.Field | pa.Schema] = None,
|
|
pip: tuple[str, ...] | list[str] = (),
|
|
env: Optional[Mapping[str, str]] = None,
|
|
python_version: Optional[str] = None,
|
|
batch_size: Optional[int] = None,
|
|
max_concurrency: Optional[int] = None,
|
|
row_timeout_seconds: Optional[float] = None,
|
|
retry_policy: Optional[FunctionRetryPolicy] = None,
|
|
gpu: bool = False,
|
|
gpu_class: Optional[str] = None,
|
|
conda: tuple[str, ...] | list[str] = (),
|
|
conda_channels: tuple[str, ...] | list[str] = (),
|
|
apt: tuple[str, ...] | list[str] = (),
|
|
image: Optional[str] = None,
|
|
code: Sequence[types.ModuleType] = (),
|
|
models: Sequence[HuggingFaceModel] = (),
|
|
):
|
|
"""Prepare a Python function or class for remote Function registration.
|
|
|
|
Input and output signatures are inferred from supported annotations. A
|
|
parameter annotated ``T`` is non-nullable and binds only a non-nullable
|
|
column; annotate ``Optional[T]`` to bind a nullable one, and it receives
|
|
``None`` for a null value. PyArrow fields are nullable unless declared
|
|
otherwise, so most columns need ``Optional``. For Arrow types annotations
|
|
cannot express precisely, pass ``input_schema``, ``output_schema``, or
|
|
both; the one omitted is inferred. A scalar output is non-nullable by
|
|
default; annotate the return as ``Optional[T]`` or ``T | None``, or pass a
|
|
nullable ``output_schema`` field, to declare a nullable scalar. The
|
|
deployment must accept nullable scalar outputs; an older server rejects
|
|
the registration. Every named-struct field may be nullable; Enterprise
|
|
preserves the struct's validity when the result is expanded into sibling
|
|
columns.
|
|
|
|
**Large values.** A ``bytes`` parameter receives the whole value, which
|
|
is held in memory with its row and counts against the server's per-row
|
|
input limit. To pass large binary values, such as video, store them in a
|
|
Blob V2 column and annotate the parameter ``BlobReader``, which streams.
|
|
|
|
**Functions and classes.** A decorated function is called once per row,
|
|
or once per batch when its parameters are annotated ``pyarrow.Array`` or
|
|
it takes one ``pyarrow.RecordBatch``. These batch annotations require both
|
|
``input_schema`` and ``output_schema`` explicitly: the annotations do not
|
|
specify Arrow element types or fields. This also applies to a class's
|
|
``__call__`` method. A decorated class is the same callable with state:
|
|
each remote instance calls ``__init__`` once, calls
|
|
``__call__`` for every row or batch of every input it processes, and calls
|
|
``close()``, if defined, once when it retires. Load models, open clients,
|
|
and build rate limiters in ``__init__``; every batch of the instance then
|
|
reuses them. An instance serves many batches of one binding, so state is
|
|
shared across them, but not across instances or workers.
|
|
|
|
Functions and ``__call__`` methods must return a value. Generators,
|
|
including async generators, are not supported; return a list to produce
|
|
multiple values for a list output. Lambdas and decorated wrappers such as
|
|
``functools.lru_cache`` are not supported. Use a named ``def`` and keep
|
|
caches inside the body or in a class instance initialized by ``__init__``.
|
|
Callable input parameters cannot have defaults; move configurable values
|
|
to class initialization fields instead.
|
|
|
|
**Initialization.** The annotated parameters of a class's ``__init__``
|
|
are the Function's initialization fields. Their values belong to each
|
|
binding, not to the Function: ``function(text=col("body"), model="small")``
|
|
binds column ``body`` and initializes every instance with ``model="small"``,
|
|
and another column can bind the same Function version with other values.
|
|
Initialization fields take booleans, integers, floats, strings, and lists
|
|
or structs of them (``Annotated[dict, pa.struct(...)]``). A parameter with
|
|
a default may be omitted from a binding; a null value then takes the
|
|
default. Use Secrets, not initialization, for credentials.
|
|
|
|
**What travels with the Function.** The artifact is a snapshot of the
|
|
decorated source plus exactly the module-level names it references:
|
|
|
|
- modules, and classes and functions importable from an installed
|
|
package, become imports; the package must be declared in ``pip`` or
|
|
``conda``;
|
|
- literals (``None``, ``bool``, ``int``, ``str``, ``bytes``, finite
|
|
``float``, and tuples containing only these literals) are inlined;
|
|
- functions and classes defined in ``__main__``, such as notebook cells,
|
|
are packaged by source, together with what they reference;
|
|
- modules and packages listed in ``code`` are shipped as Python source and
|
|
imported normally, so helpers shared by several Functions live in one
|
|
place.
|
|
|
|
A module imported by the packaged source -- including a direct ``import``
|
|
inside a function or method body -- draws a warning at registration when
|
|
the client cannot resolve its top-level module and declares no package
|
|
dependencies, or when it resolves to local source outside ``code`` and
|
|
no declared requirement matches the imported module name. The warning
|
|
does not assert that the remote environment lacks the module:
|
|
``pip``/``conda`` distribution names need not match the modules they
|
|
install, and the client cannot inspect the contents of a base or custom
|
|
image. Warnings follow normal Python warning filters and may be promoted
|
|
to errors. Declare every third-party import in ``pip`` or ``conda`` even
|
|
when it happens to be installed on the registration host.
|
|
Other global values, including Arrow types and schemas, lists, dictionaries,
|
|
sets, and NumPy arrays, cannot be captured as literals. Define them inside
|
|
the function or class body instead.
|
|
Closures, lambdas, nested definitions, mutable module-level objects, and a
|
|
non-standard ``__builtins__`` are rejected where they can be seen and
|
|
otherwise unsupported.
|
|
|
|
**Namespace restrictions.** The packaged source cannot import ``sys``,
|
|
``builtins``, ``importlib``, ``inspect``, ``gc``, ``ctypes``, or ``types``,
|
|
including submodules such as ``ctypes.util`` and ``importlib.metadata``.
|
|
Referencing a module-level alias for one of these modules is also rejected.
|
|
References to ``globals``, ``locals``, ``vars``, ``eval``, ``exec``,
|
|
``compile``, or ``__import__`` are rejected as dynamic namespace access.
|
|
These restrictions are conservative: even ordinary uses such as
|
|
``ctypes.util.find_library`` or ``sys.version`` are blocked because these
|
|
modules can reach names outside the reconstructed Function namespace.
|
|
Move such code into an importable helper module supplied with ``code=``,
|
|
``pip=``, or ``conda=``, then import and call the helper from the Function.
|
|
Helpers imported normally keep their own module namespace; these checks do
|
|
not scan their bodies. A helper defined in ``__main__`` is packaged by source
|
|
and is subject to the same checks.
|
|
|
|
Changing any packaged source, including a module in ``code``, produces a
|
|
new Function version; the environment built from ``pip`` or ``conda`` is
|
|
reused.
|
|
|
|
Parameters
|
|
----------
|
|
function : function or class, optional
|
|
The synchronous callable, or the class whose instances are called. For
|
|
a function, annotate every parameter as ``pyarrow.Array`` to opt into
|
|
vectorized execution; a matching return annotation documents that the
|
|
callable returns an Arrow array.
|
|
name : str, optional
|
|
The remote Function name. Defaults to the callable name.
|
|
Must start with an ASCII letter or underscore and contain only ASCII
|
|
letters, digits, and underscores.
|
|
input_schema : pyarrow.Schema, optional
|
|
Explicit input fields in the exact order of the callable parameters.
|
|
Required for batch annotations. When omitted, inputs are inferred from
|
|
the parameter annotations.
|
|
output_schema : pyarrow.DataType, pyarrow.Field, or pyarrow.Schema, optional
|
|
Explicit scalar or named-struct output. A scalar field's ``nullable``
|
|
flag is preserved; a top-level named-struct field and a Blob v2 output
|
|
must be non-nullable. Required for batch annotations. When omitted, the
|
|
output is inferred from the return annotation.
|
|
pip : sequence of str, optional
|
|
Pip requirements for the remote environment. Syntax is validated at
|
|
decoration time, including extras and environment markers. Named direct
|
|
references and bare package or git URLs are supported.
|
|
conda : sequence of str, optional
|
|
Conda packages for the remote environment, instead of ``pip``. MatchSpec
|
|
syntax is validated at decoration time, including channels and builds.
|
|
conda_channels : sequence of str, optional
|
|
Conda channels in priority order; requires ``conda``.
|
|
apt : sequence of str, optional
|
|
Debian packages (``"name"`` or ``"name=version"``) added to the remote
|
|
image before the Python environment, for shared libraries pip and
|
|
conda cannot provide, such as FFmpeg's for ``torchcodec``. Packages
|
|
are unpacked without running their maintainer scripts. Works with
|
|
``pip`` or ``conda``; requires image execution.
|
|
image : str, optional
|
|
A digest-pinned registry image (``"host/repository@sha256:<digest>"``)
|
|
the remote environment is built on instead of the default base. Pip
|
|
requirements install only what the image lacks.
|
|
env : mapping of str to str, optional
|
|
Environment variables included in the Function definition. Not for
|
|
credentials -- these are ordinary configuration, stored with the
|
|
Function and visible wherever it is. Prefer initialization for
|
|
values that differ between bindings.
|
|
python_version : str, optional
|
|
Remote Python major/minor version. Defaults to the client version.
|
|
batch_size : int, optional
|
|
Maximum number of input rows supplied in one execution batch. Must be
|
|
between 1 and 4294967295. Remote execution may use smaller batches.
|
|
When omitted, the service chooses the batch size adaptively.
|
|
max_concurrency : int, optional
|
|
Maximum user-code calls in flight for one Function within one logical
|
|
execution. Must be between 1 and 4294967295. A refresh can override
|
|
this default for that execution only. Omitting it preserves the
|
|
existing behavior. This is not an account-wide quota.
|
|
row_timeout_seconds : float, optional
|
|
Positive, finite deadline in seconds for each Python call: one row for
|
|
scalar Functions and one batch for vectorized Functions. This does not
|
|
govern class initialization. When omitted, the service applies its
|
|
default per-call budget.
|
|
retry_policy : FunctionRetryPolicy, optional
|
|
Retry selected failed calls inside the Function, with bounded attempts
|
|
and backoff. A refresh may override this default. Retrying a partial
|
|
call must be safe; completed calls are not repeated. For batch Functions,
|
|
skip after exhaustion applies to the whole failed call.
|
|
gpu : bool, default False
|
|
Whether every remote execution requires a GPU. The execution platform
|
|
selects one compatible GPU for each worker. The requirement is part of
|
|
the immutable Function version.
|
|
gpu_class : str, optional
|
|
Operator-defined GPU class for every execution of this Function, such
|
|
as ``"a10g"`` or ``"h100"``. Requires ``gpu=True`` and follows the
|
|
lowercase 63-character DNS-label rule. The class is part of the
|
|
immutable Function version; per-refresh aggregate GPU budgets are
|
|
selected through ``refresh_column`` instead of registration.
|
|
code : sequence of module, optional
|
|
Top-level modules or packages shipped with the Function and importable
|
|
by their own names. Package resources, including binary data and
|
|
vendored subpackages, retain their relative paths and exact bytes.
|
|
Source and resource changes both create a new Function version.
|
|
Caches and hidden paths are excluded. Symlinks and unsafe paths are
|
|
skipped with a warning. Compiled dependencies must match the remote
|
|
runtime; use ``pip=`` or ``conda=`` for platform-specific distributions
|
|
and ``models=`` for large model snapshots. Resource bundles require
|
|
server support for the ``python_bundle.resources`` field.
|
|
models : sequence of HuggingFaceModel, optional
|
|
Immutable Hugging Face repository snapshots prepared in the worker
|
|
node cache before user code starts. A Function may declare a
|
|
repository once. Different revisions of one repository use different
|
|
Function versions. Model Functions run offline with a private
|
|
``HF_HOME``; bind ``HF_TOKEN`` through Secrets when the repository
|
|
requires authentication.
|
|
|
|
Returns
|
|
-------
|
|
FunctionDefinition
|
|
A callable definition accepted by
|
|
:meth:`lancedb.db.DBConnection.create_function`,
|
|
:meth:`lancedb.db.AsyncConnection.create_function_async` and
|
|
:meth:`lancedb.db.DBConnection.create_function_async`.
|
|
|
|
Examples
|
|
--------
|
|
>>> from lancedb import function
|
|
>>> @function(pip=["numpy==2.2.0"])
|
|
... def score(value: float) -> float:
|
|
... return value * 2
|
|
>>> score(1.5)
|
|
3.0
|
|
>>> import pyarrow as pa
|
|
>>> @function(
|
|
... input_schema=pa.schema([pa.field("value", pa.int64(), nullable=False)]),
|
|
... output_schema=pa.int64(),
|
|
... )
|
|
... def batch_identity(value: pa.Array) -> pa.Array:
|
|
... return value
|
|
>>> batch_identity(pa.array([1, 2, 3])).to_pylist()
|
|
[1, 2, 3]
|
|
>>> @function(pip=["cupy-cuda12x"], gpu=True, gpu_class="a10g")
|
|
... def gpu_score(value: int) -> int:
|
|
... return value * 2
|
|
>>> gpu_score.registration_request.runtime.gpu
|
|
True
|
|
>>> gpu_score.registration_request.runtime.gpu_class
|
|
'a10g'
|
|
>>> @function
|
|
... class scale:
|
|
... def __init__(self, factor: float, offset: float = 0.0):
|
|
... self.factor, self.offset = factor, offset
|
|
... def __call__(self, value: float) -> float:
|
|
... return value * self.factor + self.offset
|
|
... def close(self):
|
|
... self.factor = 0.0
|
|
>>> scale_fn = scale(factor=2.0)
|
|
>>> scale_fn(1.5)
|
|
3.0
|
|
>>> scale_fn(2.0)
|
|
4.0
|
|
>>> scale_fn.close()
|
|
>>> [field.name for field in scale.registration_request.signature.initialization]
|
|
['factor', 'offset']
|
|
>>> import lancedb
|
|
>>> @function(
|
|
... input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
|
|
... output_schema=lancedb.blob("copy", nullable=False),
|
|
... )
|
|
... def copy_blobs(image: pa.Array) -> pa.Array:
|
|
... return image
|
|
|
|
**Local execution.** A function definition is callable with its decorated
|
|
input signature and runs the original function in the current Python
|
|
environment. A class definition is called with its initialization
|
|
arguments and constructs one instance; call that instance for row inputs,
|
|
reuse it as needed, and call ``close`` if it defines that method. This path
|
|
does not create a registered
|
|
[FunctionVersion][lancedb.functions.FunctionVersion], build the declared
|
|
remote environment, resolve Secrets, or run the remote Arrow adapter or
|
|
sandbox. A registered ``FunctionVersion`` call is different: it binds
|
|
``col(...)`` inputs and returns a remote
|
|
[FunctionApplication][lancedb.functions.FunctionApplication].
|
|
|
|
```python
|
|
scaled = scale(factor=2.0)
|
|
try:
|
|
assert scaled(1.5) == 3.0
|
|
assert scaled(2.0) == 4.0
|
|
finally:
|
|
scaled.close()
|
|
```
|
|
|
|
Blob v2 input is exposed as ``pyarrow.LargeBinaryArray`` in vectorized
|
|
mode (and as ``bytes`` in row mode). Return a ``LargeBinaryArray`` with the
|
|
declared nullability. Use :func:`lancedb.blob` in the explicit schemas;
|
|
plain ``pyarrow.large_binary()`` does not carry Blob v2 semantics.
|
|
|
|
A Blob v2 field may sit at the top level or inside a struct in inputs and
|
|
outputs. Only outputs may nest it in a list or large list (for example
|
|
``pa.list_(lancedb.blob("item"))``, returned as a list of ``bytes``);
|
|
fixed-size lists and maps are not supported. List-nested Blob v2 values
|
|
are queryable as Blob descriptors, but the row-id blob fetch APIs and
|
|
``blob_mode="bytes"`` cannot fetch payloads inside lists.
|
|
"""
|
|
|
|
def decorate(target: Any) -> FunctionDefinition:
|
|
return FunctionDefinition(
|
|
target,
|
|
name=name,
|
|
input_schema=input_schema,
|
|
output_schema=output_schema,
|
|
pip=tuple(pip),
|
|
env={} if env is None else env,
|
|
python_version=python_version,
|
|
batch_size=batch_size,
|
|
max_concurrency=max_concurrency,
|
|
row_timeout_seconds=row_timeout_seconds,
|
|
retry_policy=retry_policy,
|
|
gpu=gpu,
|
|
gpu_class=gpu_class,
|
|
conda=tuple(conda),
|
|
conda_channels=tuple(conda_channels),
|
|
apt=tuple(apt),
|
|
image=image,
|
|
code=tuple(code),
|
|
models=tuple(models),
|
|
)
|
|
|
|
if target is None:
|
|
return decorate
|
|
return decorate(target)
|
|
|
|
|
|
__all__ = [
|
|
"AssignmentMapping",
|
|
"ApplicationInput",
|
|
"FunctionApplication",
|
|
"FunctionArtifact",
|
|
"FunctionArtifactContent",
|
|
"FunctionArtifactRequest",
|
|
"FunctionBinding",
|
|
"FunctionOutput",
|
|
"FunctionParameter",
|
|
"FunctionRegistrationRequest",
|
|
"FunctionRetryPolicy",
|
|
"FunctionResultField",
|
|
"FunctionSignature",
|
|
"FunctionImage",
|
|
"FunctionVersion",
|
|
"FunctionVersionRef",
|
|
"HuggingFaceModel",
|
|
"InputBinding",
|
|
"LiteralInputBinding",
|
|
"OutputMapping",
|
|
"PythonEnvironmentSpec",
|
|
"PythonAdapterSpec",
|
|
"PythonRuntimeSpec",
|
|
"RefreshColumnResult",
|
|
"FunctionDefinition",
|
|
]
|
|
|
|
|
|
def udf(*args: Any, **kwargs: Any) -> Any:
|
|
"""Deprecated alias of :func:`function`."""
|
|
warnings.warn(
|
|
"lancedb.udf is deprecated; use lancedb.function",
|
|
DeprecationWarning,
|
|
stacklevel=2,
|
|
)
|
|
return function(*args, **kwargs)
|
|
|
|
|
|
UDF_DEFINITION_DEPRECATION = (
|
|
"lancedb.UdfDefinition is deprecated; use lancedb.FunctionDefinition"
|
|
)
|
|
|
|
|
|
def __getattr__(name: str) -> Any:
|
|
if name == "UdfDefinition":
|
|
warnings.warn(UDF_DEFINITION_DEPRECATION, DeprecationWarning, stacklevel=2)
|
|
return FunctionDefinition
|
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
|
|
|
|
if TYPE_CHECKING:
|
|
UdfDefinition = FunctionDefinition
|