mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-24 23:18:39 +00:00
feat: add grouped function column bindings (#3994)
Function applications from the canonical remote contract cannot currently declare scalar or grouped computed-column outputs atomically. This adds the remote-only declaration contract for scalar, struct-as-one-column, and expanded named-struct outputs. It validates result mappings, fixes exact input/output Arrow schemas in the request, persists grouped sibling metadata, and keeps local Function execution unsupported. Unknown newer application or binding metadata remains readable, while schema-changing mutations fail closed instead of rewriting it. Stable Lance field IDs are deliberately not a declaration prerequisite in this slice. Inputs bind by parameter name and field path; Sophon remains responsible for exact-version validation, atomic all-NULL sibling creation, binding identity and revision allocation, and persisted output identities.
This commit is contained in:
@@ -341,6 +341,9 @@ class Table:
|
||||
async def add_computed_columns(
|
||||
self, columns: list[tuple[str, str]]
|
||||
) -> AddColumnsResult: ...
|
||||
async def add_function_columns(
|
||||
self, application_json: str, output_name: Optional[str]
|
||||
) -> AddColumnsResult: ...
|
||||
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
|
||||
async def refresh_column_async(self, column: str) -> Job: ...
|
||||
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
|
||||
|
||||
@@ -123,6 +123,23 @@ class _RemoteValue(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
class _OpenRemoteValue(_RemoteValue):
|
||||
"""Forward-readable value whose extras stay out of canonical encoding."""
|
||||
|
||||
if _PYDANTIC_V2:
|
||||
model_config = {"extra": "allow", "frozen": True}
|
||||
else:
|
||||
|
||||
class Config:
|
||||
allow_mutation = False
|
||||
extra = "allow"
|
||||
|
||||
def _unknown_field_names(self) -> set[str]:
|
||||
if _PYDANTIC_V2:
|
||||
return set((self.__pydantic_extra__ or {}).keys())
|
||||
return set(self.__dict__) - set(self.__fields__)
|
||||
|
||||
|
||||
class FunctionArtifact(_RemoteValue):
|
||||
"""Content-addressed Python artifact identity."""
|
||||
|
||||
@@ -137,13 +154,13 @@ class FunctionParameter(_RemoteValue):
|
||||
nullable: bool
|
||||
|
||||
|
||||
class FunctionResultField(_RemoteValue):
|
||||
class FunctionResultField(_OpenRemoteValue):
|
||||
name: str
|
||||
arrow_type: str
|
||||
nullable: bool
|
||||
|
||||
|
||||
class FunctionOutput(_RemoteValue):
|
||||
class FunctionOutput(_OpenRemoteValue):
|
||||
"""Scalar or ordered named-struct output; unknown kinds remain decodable."""
|
||||
|
||||
kind: str
|
||||
@@ -211,12 +228,12 @@ class FunctionVersion(_RemoteValue):
|
||||
created_at: str
|
||||
|
||||
|
||||
class FunctionVersionRef(_RemoteValue):
|
||||
class FunctionVersionRef(_OpenRemoteValue):
|
||||
name: str
|
||||
version: str
|
||||
|
||||
|
||||
class ApplicationInput(_RemoteValue):
|
||||
class ApplicationInput(_OpenRemoteValue):
|
||||
"""One parameter value.
|
||||
|
||||
Slice 1 freezes integers, strings, booleans, nulls, arrays, and objects.
|
||||
@@ -234,7 +251,7 @@ class ApplicationInput(_RemoteValue):
|
||||
return _validate_literal(value)
|
||||
|
||||
|
||||
class FunctionApplication(_RemoteValue):
|
||||
class FunctionApplication(_OpenRemoteValue):
|
||||
"""Immutable pre-declaration application of an exact Function version."""
|
||||
|
||||
function: FunctionVersionRef
|
||||
@@ -243,6 +260,33 @@ class FunctionApplication(_RemoteValue):
|
||||
group_id: str
|
||||
columns: Mapping[str, str] = Field(default_factory=dict)
|
||||
|
||||
def _known_dict(self) -> dict[str, Any]:
|
||||
value = super()._known_dict()
|
||||
for name in self._unknown_field_names():
|
||||
value.pop(name, None)
|
||||
return value
|
||||
|
||||
def _ensure_declarable(self) -> None:
|
||||
unknown = {f"application.{name}" for name in self._unknown_field_names()}
|
||||
unknown.update(
|
||||
f"function.{name}" for name in self.function._unknown_field_names()
|
||||
)
|
||||
for index, input_value in enumerate(self.inputs):
|
||||
unknown.update(
|
||||
f"inputs[{index}].{name}" for name in input_value._unknown_field_names()
|
||||
)
|
||||
unknown.update(f"output.{name}" for name in self.output._unknown_field_names())
|
||||
for index, field in enumerate(self.output.fields):
|
||||
unknown.update(
|
||||
f"output.fields[{index}].{name}"
|
||||
for name in field._unknown_field_names()
|
||||
)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
"Function application contains fields from a newer contract: "
|
||||
f"{sorted(unknown)!r}"
|
||||
)
|
||||
|
||||
def rename(self, *, columns: Mapping[str, str]) -> FunctionApplication:
|
||||
"""Return a copy with result-field to table-column aliases."""
|
||||
if self.output.kind != "named_struct":
|
||||
@@ -293,6 +337,8 @@ class FunctionBinding(_RemoteValue):
|
||||
group_id: str
|
||||
inputs: tuple[InputBinding, ...]
|
||||
outputs: tuple[OutputMapping, ...]
|
||||
input_schema: Optional[Mapping[str, Any]] = None
|
||||
output_schema: Optional[Mapping[str, Any]] = None
|
||||
|
||||
|
||||
class RefreshColumnResult(_RemoteValue):
|
||||
|
||||
@@ -49,6 +49,7 @@ from lancedb.index import (
|
||||
LabelList,
|
||||
)
|
||||
from lancedb.job import Job
|
||||
from lancedb.functions import FunctionApplication
|
||||
from lancedb.remote.db import LOOP
|
||||
from lancedb.table import IndexConfigType, KNOWN_METRICS
|
||||
import pyarrow as pa
|
||||
@@ -960,7 +961,9 @@ class RemoteTable(Table):
|
||||
|
||||
def add_columns(
|
||||
self,
|
||||
transforms: Dict[str, str] | None = None,
|
||||
transforms: Dict[str, str | FunctionApplication]
|
||||
| FunctionApplication
|
||||
| None = None,
|
||||
*,
|
||||
computed: Dict[str, str] | None = None,
|
||||
) -> AddColumnsResult:
|
||||
|
||||
@@ -72,6 +72,7 @@ from .index import (
|
||||
FTS,
|
||||
)
|
||||
from .expr import Expr
|
||||
from .functions import FunctionApplication
|
||||
from .merge import LanceMergeInsertBuilder
|
||||
from .pydantic import LanceModel, model_to_dict
|
||||
from .query import (
|
||||
@@ -1942,7 +1943,8 @@ class Table(ABC):
|
||||
@abstractmethod
|
||||
def add_columns(
|
||||
self,
|
||||
transforms: Dict[str, str]
|
||||
transforms: Dict[str, str | FunctionApplication]
|
||||
| FunctionApplication
|
||||
| pa.Field
|
||||
| List[pa.Field]
|
||||
| pa.Schema
|
||||
@@ -1955,13 +1957,21 @@ class Table(ABC):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema
|
||||
transforms: Dict[str, str | FunctionApplication], FunctionApplication,
|
||||
pa.Field, List[pa.Field], pa.Schema
|
||||
A map of column name to a SQL expression to use to calculate the
|
||||
value of the new column. These expressions will be evaluated for
|
||||
each row in the table, and can reference existing columns.
|
||||
Alternatively, a pyarrow Field or Schema can be provided to add
|
||||
new columns with the specified data types. The new columns will
|
||||
be initialized with null values.
|
||||
|
||||
A mapping with one ``FunctionApplication`` value keeps its scalar
|
||||
or named-struct result in the named table column. A bare
|
||||
named-struct application expands its ordered result fields as one
|
||||
atomic sibling group; aliases come from ``rename(columns=...)``.
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
A map of column name to a SQL expression defining the column. The
|
||||
column's type and inputs are derived from the expression, so no
|
||||
@@ -4056,9 +4066,10 @@ class LanceTable(Table):
|
||||
|
||||
def add_columns(
|
||||
self,
|
||||
transforms: Dict[str, str]
|
||||
| pa.field
|
||||
| List[pa.field]
|
||||
transforms: Dict[str, str | FunctionApplication]
|
||||
| FunctionApplication
|
||||
| pa.Field
|
||||
| List[pa.Field]
|
||||
| pa.Schema
|
||||
| None = None,
|
||||
*,
|
||||
@@ -5992,9 +6003,10 @@ class AsyncTable:
|
||||
|
||||
async def add_columns(
|
||||
self,
|
||||
transforms: dict[str, str]
|
||||
| pa.field
|
||||
| List[pa.field]
|
||||
transforms: dict[str, str | FunctionApplication]
|
||||
| FunctionApplication
|
||||
| pa.Field
|
||||
| List[pa.Field]
|
||||
| pa.Schema
|
||||
| None = None,
|
||||
*,
|
||||
@@ -6005,12 +6017,19 @@ class AsyncTable:
|
||||
|
||||
Parameters
|
||||
----------
|
||||
transforms: Dict[str, str]
|
||||
transforms: Dict[str, str | FunctionApplication] or FunctionApplication
|
||||
A map of column name to a SQL expression to use to calculate the
|
||||
value of the new column. These expressions will be evaluated for
|
||||
each row in the table, and can reference existing columns.
|
||||
Alternatively, you can pass a pyarrow field or schema to add
|
||||
new columns with NULLs.
|
||||
|
||||
A mapping with one ``FunctionApplication`` value keeps its scalar
|
||||
or named-struct result in the named table column. A bare
|
||||
named-struct application expands its ordered result fields as one
|
||||
atomic sibling group; aliases come from ``rename(columns=...)``.
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
A map of column name to a SQL expression defining the column. The
|
||||
column's type and inputs are derived from the expression.
|
||||
@@ -6034,6 +6053,32 @@ class AsyncTable:
|
||||
version: the new version number of the table after adding columns.
|
||||
|
||||
"""
|
||||
function_application = None
|
||||
function_output_name = None
|
||||
if isinstance(transforms, FunctionApplication):
|
||||
function_application = transforms
|
||||
elif isinstance(transforms, dict) and any(
|
||||
isinstance(value, FunctionApplication) for value in transforms.values()
|
||||
):
|
||||
if len(transforms) != 1 or not all(
|
||||
isinstance(value, FunctionApplication) for value in transforms.values()
|
||||
):
|
||||
raise ValueError(
|
||||
"one add_columns call declares exactly one Function sibling group"
|
||||
)
|
||||
function_output_name, function_application = next(iter(transforms.items()))
|
||||
|
||||
if function_application is not None:
|
||||
if computed:
|
||||
raise ValueError(
|
||||
"add_columns cannot mix a Function application with SQL "
|
||||
"computed columns"
|
||||
)
|
||||
function_application._ensure_declarable()
|
||||
return await self._inner.add_function_columns(
|
||||
function_application.to_canonical_json(), function_output_name
|
||||
)
|
||||
|
||||
if isinstance(transforms, pa.Field):
|
||||
transforms = [transforms]
|
||||
if isinstance(transforms, list) and all(
|
||||
|
||||
@@ -14,6 +14,7 @@ from lancedb.functions import (
|
||||
PythonRuntimeSpec,
|
||||
RefreshColumnResult,
|
||||
)
|
||||
from lancedb.table import AsyncTable
|
||||
|
||||
|
||||
FIXTURES = (
|
||||
@@ -166,6 +167,8 @@ def test_binding_and_refresh_result_keep_stable_remote_fields():
|
||||
assert binding.revision == 3
|
||||
assert binding.function.version == "fv_01K3TEXT"
|
||||
assert [output.output_ordinal for output in binding.outputs] == [0, 1]
|
||||
assert binding.input_schema is not None
|
||||
assert binding.output_schema is not None
|
||||
|
||||
result = RefreshColumnResult.from_json(
|
||||
json.dumps(job_result("remote_refresh_job.json"))
|
||||
@@ -223,3 +226,86 @@ def test_canonical_client_values_contain_secret_names_only():
|
||||
canonical = json.loads(version.to_canonical_json())
|
||||
assert canonical["required_secrets"] == ["HF_TOKEN"]
|
||||
assert_no_secret_values(canonical)
|
||||
|
||||
|
||||
class _FunctionDeclarationInner:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
async def add_function_columns(self, application_json, output_name):
|
||||
self.calls.append((json.loads(application_json), output_name))
|
||||
return "declared"
|
||||
|
||||
|
||||
def known_application() -> FunctionApplication:
|
||||
value = json.loads(fixture("remote_function_application.json"))
|
||||
value.pop("future_application")
|
||||
return FunctionApplication(**value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically():
|
||||
inner = _FunctionDeclarationInner()
|
||||
table = AsyncTable(inner)
|
||||
application = known_application()
|
||||
|
||||
result = await table.add_columns(
|
||||
{"features": application._copy(update={"columns": {}})}
|
||||
)
|
||||
assert result == "declared"
|
||||
assert inner.calls[-1][1] == "features"
|
||||
|
||||
bare = application._copy(update={"columns": {}}).rename(
|
||||
columns={"normalized_text": "search_text"}
|
||||
)
|
||||
result = await table.add_columns(bare)
|
||||
assert result == "declared"
|
||||
assert inner.calls[-1][1] is None
|
||||
assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_columns_rejects_mixed_groups_and_unknown_newer_application():
|
||||
inner = _FunctionDeclarationInner()
|
||||
table = AsyncTable(inner)
|
||||
application = known_application()
|
||||
|
||||
with pytest.raises(ValueError, match="exactly one Function sibling group"):
|
||||
await table.add_columns({"a": application, "b": application})
|
||||
|
||||
future = json.loads(fixture("remote_function_application.json"))
|
||||
application = FunctionApplication(**future)
|
||||
with pytest.raises(ValueError, match="newer contract"):
|
||||
await table.add_columns(application)
|
||||
|
||||
future.pop("future_application")
|
||||
future["output"]["assignment"] = "cell_flag"
|
||||
application = FunctionApplication(**future)
|
||||
assert "assignment" not in json.loads(application.to_canonical_json())["output"]
|
||||
with pytest.raises(ValueError, match="output.assignment"):
|
||||
await table.add_columns(application)
|
||||
assert inner.calls == []
|
||||
|
||||
|
||||
def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
|
||||
scalar = FunctionApplication.from_json(
|
||||
json.dumps(
|
||||
{
|
||||
"function": {"name": "embed", "version": "fv_exact"},
|
||||
"inputs": [],
|
||||
"output": {
|
||||
"kind": "scalar",
|
||||
"arrow_type": "list<float32>",
|
||||
"nullable": False,
|
||||
},
|
||||
"group_id": "fg_scalar",
|
||||
}
|
||||
)
|
||||
)
|
||||
with pytest.raises(ValueError, match="named-struct"):
|
||||
scalar.rename(columns={"value": "embedding"})
|
||||
|
||||
application = known_application()._copy(update={"columns": {}})
|
||||
renamed = application.rename(columns={"normalized_text": "search_text"})
|
||||
assert dict(application.columns) == {}
|
||||
assert dict(renamed.columns) == {"normalized_text": "search_text"}
|
||||
|
||||
@@ -1551,6 +1551,24 @@ impl Table {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn add_function_columns(
|
||||
self_: PyRef<'_, Self>,
|
||||
application_json: String,
|
||||
output_name: Option<String>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let application =
|
||||
lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
let builder = match output_name {
|
||||
Some(name) => inner.add_columns().function_as(name, application),
|
||||
None => inner.add_columns().function(application),
|
||||
};
|
||||
let result = builder.execute().await.infer_error()?;
|
||||
Ok(AddColumnsResult::from(result))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn refresh_column(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.inner_ref()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
|
||||
@@ -74,6 +74,46 @@ fn validate_literal(value: &Value) -> Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
fn has_unknown_keys(value: &Value, allowed: &[&str]) -> bool {
|
||||
value
|
||||
.as_object()
|
||||
.is_some_and(|object| object.keys().any(|key| !allowed.contains(&key.as_str())))
|
||||
}
|
||||
|
||||
fn application_has_unknown_nested_fields(value: &Value) -> bool {
|
||||
let Some(application) = value.as_object() else {
|
||||
return false;
|
||||
};
|
||||
if application
|
||||
.get("function")
|
||||
.is_some_and(|value| has_unknown_keys(value, &["name", "version"]))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if application
|
||||
.get("inputs")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|inputs| {
|
||||
inputs
|
||||
.iter()
|
||||
.any(|input| has_unknown_keys(input, &["parameter", "kind", "value"]))
|
||||
})
|
||||
{
|
||||
return true;
|
||||
}
|
||||
application.get("output").is_some_and(|output| {
|
||||
has_unknown_keys(output, &["kind", "arrow_type", "nullable", "fields"])
|
||||
|| output
|
||||
.get("fields")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|fields| {
|
||||
fields
|
||||
.iter()
|
||||
.any(|field| has_unknown_keys(field, &["name", "arrow_type", "nullable"]))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
macro_rules! impl_json {
|
||||
($type:ty) => {
|
||||
impl $type {
|
||||
@@ -358,6 +398,10 @@ pub struct FunctionApplication {
|
||||
group_id: String,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
columns: BTreeMap<String, String>,
|
||||
#[serde(default, flatten, skip_serializing)]
|
||||
unknown_fields: BTreeMap<String, Value>,
|
||||
#[serde(default, skip)]
|
||||
unknown_nested_fields: bool,
|
||||
}
|
||||
|
||||
impl FunctionApplication {
|
||||
@@ -380,10 +424,18 @@ impl FunctionApplication {
|
||||
pub fn columns(&self) -> &BTreeMap<String, String> {
|
||||
&self.columns
|
||||
}
|
||||
/// Whether a newer writer attached application fields this client cannot
|
||||
/// validate. Such applications remain readable but must not be declared.
|
||||
pub fn has_unknown_fields(&self) -> bool {
|
||||
!self.unknown_fields.is_empty() || self.unknown_nested_fields
|
||||
}
|
||||
|
||||
/// Decode a remote application after validating the Slice 1 literal domain.
|
||||
pub fn from_json(json: &str) -> Result<Self> {
|
||||
let application: Self = from_json(json)?;
|
||||
let value: Value = from_json(json)?;
|
||||
let has_unknown_nested_fields = application_has_unknown_nested_fields(&value);
|
||||
let mut application: Self = serde_json::from_value(value).map_err(invalid_json)?;
|
||||
application.unknown_nested_fields = has_unknown_nested_fields;
|
||||
application
|
||||
.inputs
|
||||
.iter()
|
||||
@@ -433,6 +485,13 @@ pub struct FunctionBinding {
|
||||
group_id: String,
|
||||
inputs: Vec<InputBinding>,
|
||||
outputs: Vec<OutputMapping>,
|
||||
/// Exact Arrow schema presented to the Function, encoded with the Lance
|
||||
/// Namespace Arrow JSON representation.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
input_schema: Option<Value>,
|
||||
/// Exact physical Arrow schema of the grouped table outputs.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
output_schema: Option<Value>,
|
||||
}
|
||||
|
||||
impl FunctionBinding {
|
||||
@@ -459,6 +518,14 @@ impl FunctionBinding {
|
||||
pub fn outputs(&self) -> &[OutputMapping] {
|
||||
&self.outputs
|
||||
}
|
||||
|
||||
pub fn input_schema(&self) -> Option<&Value> {
|
||||
self.input_schema.as_ref()
|
||||
}
|
||||
|
||||
pub fn output_schema(&self) -> Option<&Value> {
|
||||
self.output_schema.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl_json!(FunctionBinding);
|
||||
|
||||
@@ -2147,6 +2147,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
self.check_mutable().await?;
|
||||
|
||||
let table_schema = self.schema().await?;
|
||||
crate::table::computed_columns::ensure_supported_function_metadata(table_schema.as_ref())?;
|
||||
let table_def = TableDefinition::try_from_rich_schema(table_schema.clone())?;
|
||||
|
||||
let num_partitions = if self.server_version.support_multipart_write() {
|
||||
@@ -2698,6 +2699,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
_read_columns: Option<Vec<String>>,
|
||||
) -> Result<AddColumnsResult> {
|
||||
self.check_mutable().await?;
|
||||
crate::table::computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
self.schema().await?.as_ref(),
|
||||
"schema evolution",
|
||||
)?;
|
||||
match transforms {
|
||||
NewColumnTransform::SqlExpressions(expressions) => {
|
||||
let body = expressions
|
||||
@@ -2746,6 +2751,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
|
||||
async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
|
||||
self.check_mutable().await?;
|
||||
crate::table::computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
self.schema().await?.as_ref(),
|
||||
"schema evolution",
|
||||
)?;
|
||||
// The server plans the declaration: expression validation, type
|
||||
// inference and the persisted binding all happen there.
|
||||
let entries = columns
|
||||
@@ -2785,6 +2794,63 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn add_function_columns(
|
||||
&self,
|
||||
application: &crate::function::FunctionApplication,
|
||||
output_name: Option<&str>,
|
||||
) -> Result<AddColumnsResult> {
|
||||
self.check_mutable().await?;
|
||||
let schema = self.schema().await?;
|
||||
let plan = crate::table::computed_columns::plan_function_application(
|
||||
schema.as_ref(),
|
||||
application,
|
||||
output_name,
|
||||
)?;
|
||||
let new_columns = plan
|
||||
.outputs
|
||||
.iter()
|
||||
.map(|output| {
|
||||
serde_json::json!({
|
||||
"name": output.output_name,
|
||||
"all_null": true,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut body = serde_json::json!({
|
||||
"new_columns": new_columns,
|
||||
"function": {
|
||||
"application": plan.application,
|
||||
"binding_metadata_version": plan.binding_metadata_version,
|
||||
"input_bindings": plan.input_bindings,
|
||||
"input_schema": plan.input_schema,
|
||||
"output_schema": plan.output_schema,
|
||||
"outputs": plan.outputs,
|
||||
},
|
||||
});
|
||||
self.apply_branch_body(&mut body);
|
||||
let request = self
|
||||
.client
|
||||
.post(&format!("/v1/table/{}/add_columns/", self.identifier))
|
||||
.json(&body);
|
||||
let (request_id, response) = self.send(request, true).await?;
|
||||
let response = self.check_table_response(&request_id, response).await?;
|
||||
let body = response.text().await.err_to_http(request_id.clone())?;
|
||||
|
||||
if body.trim().is_empty() {
|
||||
return Ok(AddColumnsResult { version: 0 });
|
||||
}
|
||||
|
||||
let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http {
|
||||
source: format!("Failed to parse add Function columns response: {e}").into(),
|
||||
request_id,
|
||||
status_code: None,
|
||||
})?;
|
||||
|
||||
self.invalidate_schema_cache();
|
||||
self.track_write_version(result.version);
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn refresh_column(&self, _column: &str) -> Result<RefreshColumnResult> {
|
||||
// The server runs a refresh as a job and does not report a fill
|
||||
// count, so the blocking form has no honest result to return.
|
||||
@@ -3810,11 +3876,14 @@ mod tests {
|
||||
assert_eq!(rename, "y");
|
||||
|
||||
if old_server {
|
||||
http::Response::builder().status(200).body("{}").unwrap()
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body("{}".to_string())
|
||||
.unwrap()
|
||||
} else {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version": 43}"#)
|
||||
.body(r#"{"version": 43}"#.to_string())
|
||||
.unwrap()
|
||||
}
|
||||
} else {
|
||||
@@ -3945,11 +4014,14 @@ mod tests {
|
||||
assert_eq!(predicate, "id in (1, 2, 3)");
|
||||
|
||||
if old_server {
|
||||
http::Response::builder().status(200).body("{}").unwrap()
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body("{}".to_string())
|
||||
.unwrap()
|
||||
} else {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version": 43}"#)
|
||||
.body(r#"{"version": 43}"#.to_string())
|
||||
.unwrap()
|
||||
}
|
||||
} else {
|
||||
@@ -6516,7 +6588,9 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_add_columns(#[case] old_server: bool) {
|
||||
let table = Table::new_with_handler("my_table", move |request| {
|
||||
if request.url().path() == "/v1/table/my_table/add_columns/" {
|
||||
if request.url().path() == "/v1/table/my_table/describe/" {
|
||||
simple_describe_response()
|
||||
} else if request.url().path() == "/v1/table/my_table/add_columns/" {
|
||||
assert_eq!(request.method(), "POST");
|
||||
assert_eq!(
|
||||
request.headers().get("Content-Type").unwrap(),
|
||||
@@ -6540,11 +6614,14 @@ mod tests {
|
||||
assert_eq!(expression, "cast(NULL as int32)");
|
||||
|
||||
if old_server {
|
||||
http::Response::builder().status(200).body("{}").unwrap()
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body("{}".to_string())
|
||||
.unwrap()
|
||||
} else {
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version": 43}"#)
|
||||
.body(r#"{"version": 43}"#.to_string())
|
||||
.unwrap()
|
||||
}
|
||||
} else {
|
||||
@@ -6569,19 +6646,22 @@ mod tests {
|
||||
/// plan; the client never types the expression itself.
|
||||
#[tokio::test]
|
||||
async fn test_add_computed_columns_sends_the_expression() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
assert_eq!(request.method(), "POST");
|
||||
assert_eq!(request.url().path(), "/v1/table/my_table/add_columns/");
|
||||
let body = request.body().unwrap().as_bytes().unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||
assert_eq!(
|
||||
value["new_columns"],
|
||||
serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
|
||||
);
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version": 7}"#)
|
||||
.unwrap()
|
||||
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => simple_describe_response(),
|
||||
"/v1/table/my_table/add_columns/" => {
|
||||
assert_eq!(request.method(), "POST");
|
||||
let body = request.body().unwrap().as_bytes().unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||
assert_eq!(
|
||||
value["new_columns"],
|
||||
serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
|
||||
);
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version": 7}"#.to_string())
|
||||
.unwrap()
|
||||
}
|
||||
path => panic!("Unexpected path: {path}"),
|
||||
});
|
||||
|
||||
let result = table
|
||||
@@ -6593,6 +6673,129 @@ mod tests {
|
||||
assert_eq!(result.version, 7);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_scalar_function_column_sends_atomic_null_declaration() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
|
||||
)
|
||||
.unwrap(),
|
||||
"/v1/table/my_table/add_columns/" => {
|
||||
let actual: serde_json::Value = serde_json::from_slice(
|
||||
request.body().unwrap().as_bytes().unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let expected: serde_json::Value = serde_json::from_str(include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json"
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(actual, expected);
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version":8}"#)
|
||||
.unwrap()
|
||||
}
|
||||
path => panic!("Unexpected path: {path}"),
|
||||
}
|
||||
});
|
||||
let application = crate::function::FunctionApplication::from_json(
|
||||
r#"{
|
||||
"function":{"name":"embed","version":"fv_01K3EXACT"},
|
||||
"inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
|
||||
"output":{"kind":"scalar","arrow_type":"list<float32>","nullable":false},
|
||||
"group_id":"fg_scalar"
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = table
|
||||
.add_columns()
|
||||
.function_as("embedding", application)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.version, 8);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_named_struct_function_expands_one_atomic_sibling_group() {
|
||||
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"version":1,"schema":{"fields":[
|
||||
{"name":"title","nullable":true,"type":{"type":"string"}},
|
||||
{"name":"body","nullable":true,"type":{"type":"string"}}
|
||||
]}}"#,
|
||||
)
|
||||
.unwrap(),
|
||||
"/v1/table/my_table/add_columns/" => {
|
||||
let actual: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
let expected: serde_json::Value = serde_json::from_str(include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_grouped_declaration_request.json"
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(actual, expected);
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"version":9}"#)
|
||||
.unwrap()
|
||||
}
|
||||
path => panic!("Unexpected path: {path}"),
|
||||
});
|
||||
let application = crate::function::FunctionApplication::from_json(
|
||||
r#"{
|
||||
"function":{"name":"text_features","version":"fv_01K3TEXT"},
|
||||
"inputs":[
|
||||
{"parameter":"title","kind":"column","value":{"path":"title"}},
|
||||
{"parameter":"body","kind":"column","value":{"path":"body"}}
|
||||
],
|
||||
"output":{"kind":"named_struct","fields":[
|
||||
{"name":"normalized_text","arrow_type":"utf8","nullable":false},
|
||||
{"name":"token_count","arrow_type":"int64","nullable":false}
|
||||
]},
|
||||
"group_id":"fg_01K3TEXT",
|
||||
"columns":{"normalized_text":"search_text"}
|
||||
}"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let result = table
|
||||
.add_columns()
|
||||
.function(application)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.version, 9);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_columns_fails_closed_on_newer_function_binding_metadata() {
|
||||
let table = Table::new_with_handler("my_table", |request| {
|
||||
match request.url().path() {
|
||||
"/v1/table/my_table/describe/" => http::Response::builder()
|
||||
.status(200)
|
||||
.body(
|
||||
r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":2,\"bindings\":[]}"}}}"#,
|
||||
)
|
||||
.unwrap(),
|
||||
path => panic!("mutation request must not be sent: {path}"),
|
||||
}
|
||||
});
|
||||
|
||||
let err = table
|
||||
.add_columns()
|
||||
.computed("doubled", "x * 2")
|
||||
.execute()
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, Error::NotSupported { .. }));
|
||||
}
|
||||
|
||||
/// A remote refresh is a server job: the async form returns its handle,
|
||||
/// and the blocking form refuses rather than invent a fill count.
|
||||
#[tokio::test]
|
||||
@@ -10911,6 +11114,7 @@ mod tests {
|
||||
.status(200)
|
||||
.body("{}".to_string())
|
||||
.unwrap(),
|
||||
"/v1/table/my_table/describe/" => simple_describe_response(),
|
||||
"/v1/table/my_table/add_columns/"
|
||||
| "/v1/table/my_table/alter_columns/"
|
||||
| "/v1/table/my_table/drop_columns/" => {
|
||||
|
||||
@@ -760,6 +760,16 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
|
||||
message: "computed columns are not supported on this table type".into(),
|
||||
})
|
||||
}
|
||||
/// Declare one immutable registered-Function output group.
|
||||
async fn add_function_columns(
|
||||
&self,
|
||||
_application: &crate::function::FunctionApplication,
|
||||
_output_name: Option<&str>,
|
||||
) -> Result<AddColumnsResult> {
|
||||
Err(Error::NotSupported {
|
||||
message: "Function columns are supported only on LanceDB Cloud and Enterprise".into(),
|
||||
})
|
||||
}
|
||||
/// Fill a computed column's unfilled rows.
|
||||
///
|
||||
/// The default returns `NotSupported`; Lance-backed tables override it.
|
||||
@@ -3158,6 +3168,7 @@ impl BaseTable for NativeTable {
|
||||
let ds = self.dataset.get().await?;
|
||||
|
||||
let table_schema = Schema::from(&ds.schema().clone());
|
||||
computed_columns::ensure_supported_function_metadata(&table_schema)?;
|
||||
computed_columns::ensure_not_written(
|
||||
&table_schema,
|
||||
add.data.schema().fields().iter().map(|f| f.name().as_str()),
|
||||
|
||||
@@ -9,6 +9,7 @@ use lance::dataset::NewColumnTransform;
|
||||
|
||||
use super::BaseTable;
|
||||
use super::schema_evolution::AddColumnsResult;
|
||||
use crate::function::FunctionApplication;
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// Adds columns to a table. See [`Table::add_columns`](super::Table::add_columns).
|
||||
@@ -16,6 +17,7 @@ pub struct AddColumnsBuilder {
|
||||
parent: Arc<dyn BaseTable>,
|
||||
transform: Option<NewColumnTransform>,
|
||||
computed: Vec<(String, String)>,
|
||||
function: Option<(FunctionApplication, Option<String>)>,
|
||||
read_columns: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
@@ -25,6 +27,7 @@ impl std::fmt::Debug for AddColumnsBuilder {
|
||||
.field("parent", &self.parent)
|
||||
.field("has_transform", &self.transform.is_some())
|
||||
.field("computed", &self.computed)
|
||||
.field("has_function", &self.function.is_some())
|
||||
.field("read_columns", &self.read_columns)
|
||||
.finish()
|
||||
}
|
||||
@@ -36,6 +39,7 @@ impl AddColumnsBuilder {
|
||||
parent,
|
||||
transform: None,
|
||||
computed: Vec::new(),
|
||||
function: None,
|
||||
read_columns: None,
|
||||
}
|
||||
}
|
||||
@@ -83,6 +87,48 @@ impl AddColumnsBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Declare every field of a named-struct Function result as one atomic
|
||||
/// sibling group. Result-field aliases come from
|
||||
/// [`FunctionApplication::columns`](crate::function::FunctionApplication::columns).
|
||||
///
|
||||
/// ```
|
||||
/// # use lancedb::Table;
|
||||
/// # use lancedb::function::FunctionApplication;
|
||||
/// # async fn declare(table: &Table, application: FunctionApplication) -> lancedb::Result<()> {
|
||||
/// table.add_columns().function(application).execute().await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn function(mut self, application: FunctionApplication) -> Self {
|
||||
self.function = Some((application, None));
|
||||
self
|
||||
}
|
||||
|
||||
/// Declare a scalar or entire named-struct Function result as one table
|
||||
/// column. The physical column starts all-null and is materialized by the
|
||||
/// remote Function refresh path.
|
||||
///
|
||||
/// ```
|
||||
/// # use lancedb::Table;
|
||||
/// # use lancedb::function::FunctionApplication;
|
||||
/// # async fn declare(table: &Table, application: FunctionApplication) -> lancedb::Result<()> {
|
||||
/// table
|
||||
/// .add_columns()
|
||||
/// .function_as("embedding", application)
|
||||
/// .execute()
|
||||
/// .await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn function_as(
|
||||
mut self,
|
||||
name: impl Into<String>,
|
||||
application: FunctionApplication,
|
||||
) -> Self {
|
||||
self.function = Some((application, Some(name.into())));
|
||||
self
|
||||
}
|
||||
|
||||
/// Limit which existing columns a [`NewColumnTransform::BatchUDF`] mapper
|
||||
/// receives. Every other transform, and a computed column, determines what
|
||||
/// it reads, so setting this alongside one is an error rather than a silent
|
||||
@@ -98,21 +144,23 @@ impl AddColumnsBuilder {
|
||||
parent,
|
||||
transform,
|
||||
computed,
|
||||
function,
|
||||
read_columns,
|
||||
} = self;
|
||||
|
||||
match (transform, computed.is_empty()) {
|
||||
(None, true) => Err(Error::InvalidInput {
|
||||
let declaration_count = usize::from(!computed.is_empty()) + usize::from(function.is_some());
|
||||
if transform.is_some() && declaration_count != 0 || declaration_count > 1 {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "add_columns cannot mix transforms, SQL computed columns, and a Function application; they cannot be added atomically in one call"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
|
||||
match (transform, computed.is_empty(), function) {
|
||||
(None, true, None) => Err(Error::InvalidInput {
|
||||
message: "add_columns requires a transform or a computed column".into(),
|
||||
}),
|
||||
// The two commit through different transforms, so one call covering
|
||||
// both would be two commits and could half-apply.
|
||||
(Some(_), false) => Err(Error::InvalidInput {
|
||||
message: "add_columns cannot mix a transform with computed columns; \
|
||||
they cannot be added atomically in one call"
|
||||
.into(),
|
||||
}),
|
||||
(Some(transform), true) => {
|
||||
(Some(transform), true, None) => {
|
||||
if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "read_columns applies only to a BatchUDF transform; \
|
||||
@@ -122,7 +170,7 @@ impl AddColumnsBuilder {
|
||||
}
|
||||
parent.add_columns(transform, read_columns).await
|
||||
}
|
||||
(None, false) => {
|
||||
(None, false, None) => {
|
||||
if read_columns.is_some() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "read_columns applies only to a BatchUDF transform; \
|
||||
@@ -132,6 +180,18 @@ impl AddColumnsBuilder {
|
||||
}
|
||||
parent.add_computed_columns(&computed).await
|
||||
}
|
||||
(None, true, Some((application, output_name))) => {
|
||||
if read_columns.is_some() {
|
||||
return Err(Error::InvalidInput {
|
||||
message: "read_columns does not apply to a Function application; its inputs are already bound"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
parent
|
||||
.add_function_columns(&application, output_name.as_deref())
|
||||
.await
|
||||
}
|
||||
_ => unreachable!("mixed add_columns modes were rejected above"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -233,6 +233,10 @@ pub(crate) async fn execute_merge_insert(
|
||||
params: MergeInsertBuilder,
|
||||
new_data: Box<dyn RecordBatchReader + Send>,
|
||||
) -> Result<MergeResult> {
|
||||
super::computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
table.schema().await?.as_ref(),
|
||||
"merge_insert",
|
||||
)?;
|
||||
match lsm::lsm_dispatch_decision(table, ¶ms).await? {
|
||||
lsm::LsmDispatch::Lsm(plan) => {
|
||||
let future =
|
||||
|
||||
@@ -169,6 +169,9 @@ fn declared_expression(dataset: &Dataset, column: &str) -> Result<String> {
|
||||
})?;
|
||||
match declaration.kind {
|
||||
ComputedColumnKind::Sql { expression } => Ok(expression),
|
||||
ComputedColumnKind::Function { .. } => Err(Error::NotSupported {
|
||||
message: "registered Function columns are refreshed only by a remote server Job".into(),
|
||||
}),
|
||||
ComputedColumnKind::Unrecognized { kind } => Err(Error::NotSupported {
|
||||
message: format!(
|
||||
"computed column '{column}' is defined by '{kind}', which this version of \
|
||||
|
||||
@@ -101,6 +101,10 @@ pub(crate) async fn execute_add_columns(
|
||||
transforms: NewColumnTransform,
|
||||
read_columns: Option<Vec<String>>,
|
||||
) -> Result<AddColumnsResult> {
|
||||
computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
table.schema().await?.as_ref(),
|
||||
"schema evolution",
|
||||
)?;
|
||||
// Declarations are admitted only through [`execute_declare`].
|
||||
match &transforms {
|
||||
NewColumnTransform::AllNulls(schema) => {
|
||||
@@ -124,6 +128,10 @@ pub(crate) async fn execute_declare(
|
||||
// checked against latest committed state, not this handle's snapshot.
|
||||
// The catch-up flag outlives unset and marks retained SSTable rows.
|
||||
table.checkout_latest().await?;
|
||||
computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
table.schema().await?.as_ref(),
|
||||
"schema evolution",
|
||||
)?;
|
||||
let catchup = table.dataset.get().await?.manifest().reader_feature_flags
|
||||
& lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP
|
||||
!= 0;
|
||||
@@ -163,6 +171,10 @@ pub(crate) async fn execute_alter_columns(
|
||||
// Nullability is not part of what an expression resolves against, so only
|
||||
// a rename or a retype can invalidate a binding.
|
||||
let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema()));
|
||||
computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
schema.as_ref(),
|
||||
"schema evolution",
|
||||
)?;
|
||||
let rebinding = alterations
|
||||
.iter()
|
||||
.filter(|alteration| alteration.rename.is_some() || alteration.data_type.is_some())
|
||||
@@ -190,6 +202,10 @@ pub(crate) async fn execute_drop_columns(
|
||||
) -> Result<DropColumnsResult> {
|
||||
table.dataset.ensure_mutable()?;
|
||||
let mut dataset = (*table.dataset.get().await?).clone();
|
||||
computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
&ArrowSchema::from(dataset.schema()),
|
||||
"schema evolution",
|
||||
)?;
|
||||
computed_columns::ensure_not_an_input(
|
||||
&std::sync::Arc::new(ArrowSchema::from(dataset.schema())),
|
||||
columns,
|
||||
@@ -215,6 +231,7 @@ pub(crate) async fn execute_update_field_metadata(
|
||||
// binding out from under a refresh. A replace on a declared column would
|
||||
// silently erase it.
|
||||
let schema = ArrowSchema::from(dataset.schema());
|
||||
computed_columns::ensure_no_function_bindings_for_mutation(&schema, "schema evolution")?;
|
||||
let declared: Vec<String> = computed_columns::computed_columns(&schema)
|
||||
.into_iter()
|
||||
.map(|declaration| declaration.name)
|
||||
|
||||
@@ -82,6 +82,10 @@ pub(crate) async fn execute_update(
|
||||
|
||||
// 1. Snapshot the current dataset
|
||||
let dataset = table.dataset.get().await?;
|
||||
super::computed_columns::ensure_no_function_bindings_for_mutation(
|
||||
&arrow_schema::Schema::from(dataset.schema()),
|
||||
"update",
|
||||
)?;
|
||||
super::computed_columns::ensure_not_written(
|
||||
&arrow_schema::Schema::from(dataset.schema()),
|
||||
update.columns.iter().map(|(name, _)| name.as_str()),
|
||||
|
||||
@@ -88,6 +88,8 @@ fn application_and_binding_match_shared_remote_goldens() {
|
||||
assert_eq!(binding.function().version, "fv_01K3TEXT");
|
||||
assert_eq!(binding.outputs()[0].output_ordinal, 0);
|
||||
assert_eq!(binding.outputs()[1].output_ordinal, 1);
|
||||
assert!(binding.input_schema().is_some());
|
||||
assert!(binding.output_schema().is_some());
|
||||
assert_eq!(
|
||||
binding.to_canonical_json().expect("canonical JSON"),
|
||||
fixture("remote_function_binding.canonical.json").trim()
|
||||
|
||||
Vendored
+1
-1
@@ -1 +1 @@
|
||||
{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}],"revision":3}
|
||||
{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}],"revision":3}
|
||||
|
||||
+12
@@ -11,5 +11,17 @@
|
||||
{"result_field": "normalized_text", "output_name": "search_text", "output_field_id": 21, "output_ordinal": 0, "arrow_type": "utf8", "nullable": false},
|
||||
{"result_field": "token_count", "output_name": "search_token_count", "output_field_id": 22, "output_ordinal": 1, "arrow_type": "int64", "nullable": false}
|
||||
],
|
||||
"input_schema": {
|
||||
"fields": [
|
||||
{"name": "title", "nullable": true, "type": {"type": "utf8"}},
|
||||
{"name": "body", "nullable": true, "type": {"type": "utf8"}}
|
||||
]
|
||||
},
|
||||
"output_schema": {
|
||||
"fields": [
|
||||
{"name": "search_text", "nullable": true, "type": {"type": "utf8"}},
|
||||
{"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}
|
||||
]
|
||||
},
|
||||
"future_binding": {"metadata_revision": 1}
|
||||
}
|
||||
|
||||
Vendored
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"new_columns": [
|
||||
{"name": "search_text", "all_null": true},
|
||||
{"name": "token_count", "all_null": true}
|
||||
],
|
||||
"function": {
|
||||
"application": {
|
||||
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
|
||||
"inputs": [
|
||||
{"parameter": "title", "kind": "column", "value": {"path": "title"}},
|
||||
{"parameter": "body", "kind": "column", "value": {"path": "body"}}
|
||||
],
|
||||
"output": {
|
||||
"kind": "named_struct",
|
||||
"fields": [
|
||||
{"name": "normalized_text", "arrow_type": "utf8", "nullable": false},
|
||||
{"name": "token_count", "arrow_type": "int64", "nullable": false}
|
||||
]
|
||||
},
|
||||
"group_id": "fg_01K3TEXT",
|
||||
"columns": {"normalized_text": "search_text"}
|
||||
},
|
||||
"binding_metadata_version": 1,
|
||||
"input_bindings": [
|
||||
{"parameter": "title", "field_path": "title", "arrow_type": "utf8", "nullable": true},
|
||||
{"parameter": "body", "field_path": "body", "arrow_type": "utf8", "nullable": true}
|
||||
],
|
||||
"input_schema": {
|
||||
"fields": [
|
||||
{"name": "title", "nullable": true, "type": {"type": "utf8"}},
|
||||
{"name": "body", "nullable": true, "type": {"type": "utf8"}}
|
||||
]
|
||||
},
|
||||
"output_schema": {
|
||||
"fields": [
|
||||
{"name": "search_text", "nullable": true, "type": {"type": "utf8"}},
|
||||
{"name": "token_count", "nullable": true, "type": {"type": "int64"}}
|
||||
]
|
||||
},
|
||||
"outputs": [
|
||||
{"result_field": "normalized_text", "output_name": "search_text", "output_ordinal": 0},
|
||||
{"result_field": "token_count", "output_name": "token_count", "output_ordinal": 1}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+41
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"new_columns": [
|
||||
{"name": "embedding", "all_null": true}
|
||||
],
|
||||
"function": {
|
||||
"application": {
|
||||
"function": {"name": "embed", "version": "fv_01K3EXACT"},
|
||||
"inputs": [
|
||||
{"parameter": "text", "kind": "column", "value": {"path": "description"}}
|
||||
],
|
||||
"output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false},
|
||||
"group_id": "fg_scalar"
|
||||
},
|
||||
"binding_metadata_version": 1,
|
||||
"input_bindings": [
|
||||
{"parameter": "text", "field_path": "description", "arrow_type": "utf8", "nullable": true}
|
||||
],
|
||||
"input_schema": {
|
||||
"fields": [
|
||||
{"name": "text", "nullable": true, "type": {"type": "utf8"}}
|
||||
]
|
||||
},
|
||||
"output_schema": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "embedding",
|
||||
"nullable": true,
|
||||
"type": {
|
||||
"type": "list",
|
||||
"fields": [
|
||||
{"name": "item", "nullable": false, "type": {"type": "float32"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"outputs": [
|
||||
{"result_field": "$value", "output_name": "embedding", "output_ordinal": 0}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user