mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-28 00:48:40 +00:00
Xuanwo
1baada89ef
A registered `FunctionVersion` has an exact identity and grouped output contract, but the Python SDK cannot currently bind it to table columns without manually constructing wire models. Calling a `FunctionVersion` with named `col(...)` references now returns one immutable `FunctionApplication` pinned to that exact version. The application preserves named-struct outputs as one sibling group, while `rename(columns=...)` defines the result-field to table-column mapping consumed by `Table.add_columns`. Derived expressions and incomplete or unknown input names fail before declaration.
390 lines
13 KiB
Python
390 lines
13 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from lancedb import col
|
|
import lancedb.functions as functions
|
|
from lancedb.functions import (
|
|
FunctionApplication,
|
|
FunctionBinding,
|
|
FunctionVersion,
|
|
PythonRuntimeSpec,
|
|
RefreshColumnResult,
|
|
)
|
|
from lancedb.table import AsyncTable
|
|
|
|
|
|
FIXTURES = (
|
|
Path(__file__).parents[3]
|
|
/ "rust"
|
|
/ "lancedb"
|
|
/ "tests"
|
|
/ "fixtures"
|
|
/ "first_class_functions"
|
|
/ "v1"
|
|
)
|
|
|
|
|
|
def fixture(name: str) -> str:
|
|
return (FIXTURES / name).read_text()
|
|
|
|
|
|
def job_result(name: str) -> dict:
|
|
return json.loads(fixture(name))["result"]
|
|
|
|
|
|
def assert_no_secret_values(value):
|
|
if isinstance(value, dict):
|
|
for key, child in value.items():
|
|
assert key not in {
|
|
"secret_value",
|
|
"secret_values",
|
|
"resolved_secret",
|
|
"resolved_secrets",
|
|
}
|
|
assert_no_secret_values(child)
|
|
elif isinstance(value, list):
|
|
for child in value:
|
|
assert_no_secret_values(child)
|
|
|
|
|
|
def test_public_function_values_are_in_api_reference():
|
|
docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
|
|
rendered = docs.read_text()
|
|
for name in functions.__all__:
|
|
assert f"::: lancedb.functions.{name}" in rendered
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("fixture_name", "canonical_name", "model", "nested_result"),
|
|
[
|
|
(
|
|
"remote_function_job.json",
|
|
"remote_function_version.canonical.json",
|
|
FunctionVersion,
|
|
True,
|
|
),
|
|
(
|
|
"remote_function_application.json",
|
|
"remote_function_application.canonical.json",
|
|
FunctionApplication,
|
|
False,
|
|
),
|
|
(
|
|
"remote_function_binding.json",
|
|
"remote_function_binding.canonical.json",
|
|
FunctionBinding,
|
|
False,
|
|
),
|
|
(
|
|
"remote_refresh_job.json",
|
|
"remote_refresh_result.canonical.json",
|
|
RefreshColumnResult,
|
|
True,
|
|
),
|
|
(
|
|
"remote_refresh_result_without_published_version.json",
|
|
"remote_refresh_result_without_published_version.canonical.json",
|
|
RefreshColumnResult,
|
|
False,
|
|
),
|
|
],
|
|
)
|
|
def test_python_and_rust_share_remote_canonical_goldens(
|
|
fixture_name, canonical_name, model, nested_result
|
|
):
|
|
value = json.loads(fixture(fixture_name))
|
|
if nested_result:
|
|
value = value["result"]
|
|
decoded = model.from_json(json.dumps(value))
|
|
assert decoded.to_canonical_json() == fixture(canonical_name).strip()
|
|
|
|
|
|
def test_function_version_identity_is_immutable_and_exact():
|
|
value = job_result("remote_function_job.json")
|
|
version = FunctionVersion.from_json(json.dumps(value))
|
|
assert version.name == "embed"
|
|
assert version.version == "fv_01K3EXACT"
|
|
assert version.required_secrets == ("HF_TOKEN",)
|
|
|
|
with pytest.raises((TypeError, ValueError)):
|
|
version.version = "fv_changed"
|
|
with pytest.raises(TypeError, match="immutable"):
|
|
version.runtime.env["TOKENIZERS_PARALLELISM"] = "true"
|
|
|
|
changed = dict(value)
|
|
changed["version"] = "fv_changed"
|
|
assert FunctionVersion(**changed) != version
|
|
|
|
|
|
def test_function_version_binds_named_columns_as_one_immutable_group():
|
|
version = FunctionVersion.from_json(
|
|
json.dumps(job_result("remote_function_job.json"))
|
|
)
|
|
|
|
application = version(text=col("documents.body"))
|
|
|
|
assert application.function.name == version.name
|
|
assert application.function.version == version.version
|
|
assert application.output is version.signature.output
|
|
assert application.group_id.startswith("fg_")
|
|
assert [
|
|
(value.parameter, value.kind, value.value["path"])
|
|
for value in application.inputs
|
|
] == [("text", "column", "documents.body")]
|
|
with pytest.raises((TypeError, ValueError)):
|
|
application.group_id = "fg_changed"
|
|
|
|
|
|
def test_function_version_binding_validates_names_and_direct_columns():
|
|
version = FunctionVersion.from_json(
|
|
json.dumps(job_result("remote_function_job.json"))
|
|
)
|
|
|
|
with pytest.raises(TypeError, match=r"missing inputs: \['text'\]"):
|
|
version()
|
|
with pytest.raises(TypeError, match=r"unknown inputs: \['body'\]"):
|
|
version(text=col("text"), body=col("body"))
|
|
with pytest.raises(TypeError, match="direct col"):
|
|
version(text=col("text").lower())
|
|
|
|
|
|
def test_function_version_keeps_named_struct_outputs_in_one_application():
|
|
value = job_result("remote_function_job.json")
|
|
value["name"] = "text_features"
|
|
value["version"] = "fv_grouped"
|
|
value["signature"] = {
|
|
"inputs": [
|
|
{"name": "title", "arrow_type": "utf8", "nullable": True},
|
|
{"name": "body", "arrow_type": "utf8", "nullable": True},
|
|
],
|
|
"output": {
|
|
"kind": "named_struct",
|
|
"fields": [
|
|
{
|
|
"name": "normalized_text",
|
|
"arrow_type": "utf8",
|
|
"nullable": False,
|
|
},
|
|
{
|
|
"name": "token_count",
|
|
"arrow_type": "int64",
|
|
"nullable": False,
|
|
},
|
|
],
|
|
},
|
|
}
|
|
version = FunctionVersion(**value)
|
|
|
|
application = version(body=col("body"), title=col("title")).rename(
|
|
columns={
|
|
"normalized_text": "search_text",
|
|
"token_count": "search_token_count",
|
|
}
|
|
)
|
|
|
|
assert [value.parameter for value in application.inputs] == ["title", "body"]
|
|
assert [field.name for field in application.output.fields] == [
|
|
"normalized_text",
|
|
"token_count",
|
|
]
|
|
assert dict(application.columns) == {
|
|
"normalized_text": "search_text",
|
|
"token_count": "search_token_count",
|
|
}
|
|
|
|
|
|
def test_unknown_fields_and_discriminators_are_forward_decodable():
|
|
value = job_result("remote_function_job.json")
|
|
value["future_version_metadata"] = {"retention_class": "catalog"}
|
|
value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"}
|
|
value["signature"]["output"]["kind"] = "future_output_shape"
|
|
|
|
version = FunctionVersion.from_json(json.dumps(value))
|
|
assert version.runtime.kind == "wasm"
|
|
assert version.runtime.python_version is None
|
|
assert version.runtime.environment is None
|
|
assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"}
|
|
assert version.signature.output.kind == "future_output_shape"
|
|
|
|
|
|
def test_function_application_uses_rename_columns_only():
|
|
application = FunctionApplication.from_json(
|
|
fixture("remote_function_application.json")
|
|
)
|
|
renamed = application.rename(columns={"normalized_text": "body_normalized"})
|
|
|
|
assert application.columns["normalized_text"] == "search_text"
|
|
assert renamed.columns["normalized_text"] == "body_normalized"
|
|
assert renamed.function == application.function
|
|
assert renamed.group_id == application.group_id
|
|
assert not hasattr(application, "rename_outputs")
|
|
with pytest.raises(TypeError, match="immutable"):
|
|
renamed.columns["normalized_text"] = "changed"
|
|
with pytest.raises(TypeError, match="immutable"):
|
|
application.inputs[0].value["path"] = "changed"
|
|
|
|
with pytest.raises(ValueError, match="unknown Function result fields"):
|
|
application.rename(columns={"missing": "search_text"})
|
|
with pytest.raises(ValueError, match="destinations must be unique"):
|
|
application.rename(columns={"normalized_text": "same", "token_count": "same"})
|
|
|
|
bare_value = json.loads(fixture("remote_function_application.json"))
|
|
bare_value.pop("columns")
|
|
bare = FunctionApplication(**bare_value)
|
|
with pytest.raises(ValueError, match="destinations must be unique"):
|
|
bare.rename(columns={"normalized_text": "token_count"})
|
|
|
|
|
|
def test_binding_and_refresh_result_keep_stable_remote_fields():
|
|
binding = FunctionBinding.from_json(fixture("remote_function_binding.json"))
|
|
assert binding.revision == 3
|
|
assert binding.function.version == "fv_01K3TEXT"
|
|
assert [output.output_ordinal for output in binding.outputs] == [0, 1]
|
|
assert binding.input_schema is not None
|
|
assert binding.output_schema is not None
|
|
|
|
result = RefreshColumnResult.from_json(
|
|
json.dumps(job_result("remote_refresh_job.json"))
|
|
)
|
|
assert result.rows_filled == result.rows_assigned
|
|
assert result.version == result.published_version
|
|
|
|
result = RefreshColumnResult.from_json(
|
|
fixture("remote_refresh_result_without_published_version.json")
|
|
)
|
|
assert result.published_version is None
|
|
assert RefreshColumnResult.from_json(result.to_canonical_json()) == result
|
|
|
|
|
|
def test_function_literal_numeric_domain_matches_rust():
|
|
with pytest.raises(ValueError, match="floating-point Function literals"):
|
|
FunctionApplication.from_json(fixture("remote_function_application_float.json"))
|
|
|
|
value = json.loads(fixture("remote_function_application_float.json"))
|
|
value["inputs"][0]["value"] = 2**64
|
|
with pytest.raises(ValueError, match="outside the canonical JSON range"):
|
|
FunctionApplication.from_json(json.dumps(value))
|
|
|
|
|
|
def test_empty_default_maps_have_stable_canonical_bytes():
|
|
runtime = PythonRuntimeSpec(
|
|
kind="python", python_version="3.12", environment={"kind": "pip"}
|
|
)
|
|
assert runtime.to_canonical_json() == (
|
|
'{"environment":{"kind":"pip"},"kind":"python","python_version":"3.12"}'
|
|
)
|
|
|
|
value = json.loads(fixture("remote_function_application.json"))
|
|
value.pop("columns")
|
|
application = FunctionApplication.from_json(json.dumps(value))
|
|
assert "columns" not in json.loads(application.to_canonical_json())
|
|
|
|
|
|
@pytest.mark.parametrize("field", ["rows_assigned", "source_version"])
|
|
def test_refresh_result_rejects_non_u64_values(field):
|
|
value = job_result("remote_refresh_job.json")
|
|
value[field] = -1
|
|
with pytest.raises(ValueError):
|
|
RefreshColumnResult.from_json(json.dumps(value))
|
|
|
|
value[field] = "1"
|
|
with pytest.raises(ValueError):
|
|
RefreshColumnResult.from_json(json.dumps(value))
|
|
|
|
|
|
def test_canonical_client_values_contain_secret_names_only():
|
|
version = FunctionVersion.from_json(
|
|
json.dumps(job_result("remote_function_job.json"))
|
|
)
|
|
canonical = json.loads(version.to_canonical_json())
|
|
assert canonical["required_secrets"] == ["HF_TOKEN"]
|
|
assert_no_secret_values(canonical)
|
|
|
|
|
|
class _FunctionDeclarationInner:
|
|
def __init__(self):
|
|
self.calls = []
|
|
|
|
async def add_function_columns(self, application_json, output_name):
|
|
self.calls.append((json.loads(application_json), output_name))
|
|
return "declared"
|
|
|
|
|
|
def known_application() -> FunctionApplication:
|
|
value = json.loads(fixture("remote_function_application.json"))
|
|
value.pop("future_application")
|
|
return FunctionApplication(**value)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically():
|
|
inner = _FunctionDeclarationInner()
|
|
table = AsyncTable(inner)
|
|
application = known_application()
|
|
|
|
result = await table.add_columns(
|
|
{"features": application._copy(update={"columns": {}})}
|
|
)
|
|
assert result == "declared"
|
|
assert inner.calls[-1][1] == "features"
|
|
|
|
bare = application._copy(update={"columns": {}}).rename(
|
|
columns={"normalized_text": "search_text"}
|
|
)
|
|
result = await table.add_columns(bare)
|
|
assert result == "declared"
|
|
assert inner.calls[-1][1] is None
|
|
assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_add_columns_rejects_mixed_groups_and_unknown_newer_application():
|
|
inner = _FunctionDeclarationInner()
|
|
table = AsyncTable(inner)
|
|
application = known_application()
|
|
|
|
with pytest.raises(ValueError, match="exactly one Function sibling group"):
|
|
await table.add_columns({"a": application, "b": application})
|
|
|
|
future = json.loads(fixture("remote_function_application.json"))
|
|
application = FunctionApplication(**future)
|
|
with pytest.raises(ValueError, match="newer contract"):
|
|
await table.add_columns(application)
|
|
|
|
future.pop("future_application")
|
|
future["output"]["assignment"] = "cell_flag"
|
|
application = FunctionApplication(**future)
|
|
assert "assignment" not in json.loads(application.to_canonical_json())["output"]
|
|
with pytest.raises(ValueError, match="output.assignment"):
|
|
await table.add_columns(application)
|
|
assert inner.calls == []
|
|
|
|
|
|
def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
|
|
scalar = FunctionApplication.from_json(
|
|
json.dumps(
|
|
{
|
|
"function": {"name": "embed", "version": "fv_exact"},
|
|
"inputs": [],
|
|
"output": {
|
|
"kind": "scalar",
|
|
"arrow_type": "list<float32>",
|
|
"nullable": False,
|
|
},
|
|
"group_id": "fg_scalar",
|
|
}
|
|
)
|
|
)
|
|
with pytest.raises(ValueError, match="named-struct"):
|
|
scalar.rename(columns={"value": "embedding"})
|
|
|
|
application = known_application()._copy(update={"columns": {}})
|
|
renamed = application.rename(columns={"normalized_text": "search_text"})
|
|
assert dict(application.columns) == {}
|
|
assert dict(renamed.columns) == {"normalized_text": "search_text"}
|