feat: add grouped function column bindings (#3994)

Function applications from the canonical remote contract cannot
currently declare scalar or grouped computed-column outputs atomically.

This adds the remote-only declaration contract for scalar,
struct-as-one-column, and expanded named-struct outputs. It validates
result mappings, fixes exact input/output Arrow schemas in the request,
persists grouped sibling metadata, and keeps local Function execution
unsupported. Unknown newer application or binding metadata remains
readable, while schema-changing mutations fail closed instead of
rewriting it.

Stable Lance field IDs are deliberately not a declaration prerequisite
in this slice. Inputs bind by parameter name and field path; Sophon
remains responsible for exact-version validation, atomic all-NULL
sibling creation, binding identity and revision allocation, and
persisted output identities.
This commit is contained in:
Xuanwo
2026-08-21 17:01:04 +08:00
committed by GitHub
parent 4ba2421254
commit 685cb01d6d
20 changed files with 1799 additions and 52 deletions
+3
View File
@@ -341,6 +341,9 @@ class Table:
async def add_computed_columns(
self, columns: list[tuple[str, str]]
) -> AddColumnsResult: ...
async def add_function_columns(
self, application_json: str, output_name: Optional[str]
) -> AddColumnsResult: ...
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
async def refresh_column_async(self, column: str) -> Job: ...
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
+51 -5
View File
@@ -123,6 +123,23 @@ class _RemoteValue(BaseModel):
)
class _OpenRemoteValue(_RemoteValue):
"""Forward-readable value whose extras stay out of canonical encoding."""
if _PYDANTIC_V2:
model_config = {"extra": "allow", "frozen": True}
else:
class Config:
allow_mutation = False
extra = "allow"
def _unknown_field_names(self) -> set[str]:
if _PYDANTIC_V2:
return set((self.__pydantic_extra__ or {}).keys())
return set(self.__dict__) - set(self.__fields__)
class FunctionArtifact(_RemoteValue):
"""Content-addressed Python artifact identity."""
@@ -137,13 +154,13 @@ class FunctionParameter(_RemoteValue):
nullable: bool
class FunctionResultField(_RemoteValue):
class FunctionResultField(_OpenRemoteValue):
name: str
arrow_type: str
nullable: bool
class FunctionOutput(_RemoteValue):
class FunctionOutput(_OpenRemoteValue):
"""Scalar or ordered named-struct output; unknown kinds remain decodable."""
kind: str
@@ -211,12 +228,12 @@ class FunctionVersion(_RemoteValue):
created_at: str
class FunctionVersionRef(_RemoteValue):
class FunctionVersionRef(_OpenRemoteValue):
name: str
version: str
class ApplicationInput(_RemoteValue):
class ApplicationInput(_OpenRemoteValue):
"""One parameter value.
Slice 1 freezes integers, strings, booleans, nulls, arrays, and objects.
@@ -234,7 +251,7 @@ class ApplicationInput(_RemoteValue):
return _validate_literal(value)
class FunctionApplication(_RemoteValue):
class FunctionApplication(_OpenRemoteValue):
"""Immutable pre-declaration application of an exact Function version."""
function: FunctionVersionRef
@@ -243,6 +260,33 @@ class FunctionApplication(_RemoteValue):
group_id: str
columns: Mapping[str, str] = Field(default_factory=dict)
def _known_dict(self) -> dict[str, Any]:
value = super()._known_dict()
for name in self._unknown_field_names():
value.pop(name, None)
return value
def _ensure_declarable(self) -> None:
unknown = {f"application.{name}" for name in self._unknown_field_names()}
unknown.update(
f"function.{name}" for name in self.function._unknown_field_names()
)
for index, input_value in enumerate(self.inputs):
unknown.update(
f"inputs[{index}].{name}" for name in input_value._unknown_field_names()
)
unknown.update(f"output.{name}" for name in self.output._unknown_field_names())
for index, field in enumerate(self.output.fields):
unknown.update(
f"output.fields[{index}].{name}"
for name in field._unknown_field_names()
)
if unknown:
raise ValueError(
"Function application contains fields from a newer contract: "
f"{sorted(unknown)!r}"
)
def rename(self, *, columns: Mapping[str, str]) -> FunctionApplication:
"""Return a copy with result-field to table-column aliases."""
if self.output.kind != "named_struct":
@@ -293,6 +337,8 @@ class FunctionBinding(_RemoteValue):
group_id: str
inputs: tuple[InputBinding, ...]
outputs: tuple[OutputMapping, ...]
input_schema: Optional[Mapping[str, Any]] = None
output_schema: Optional[Mapping[str, Any]] = None
class RefreshColumnResult(_RemoteValue):
+4 -1
View File
@@ -49,6 +49,7 @@ from lancedb.index import (
LabelList,
)
from lancedb.job import Job
from lancedb.functions import FunctionApplication
from lancedb.remote.db import LOOP
from lancedb.table import IndexConfigType, KNOWN_METRICS
import pyarrow as pa
@@ -960,7 +961,9 @@ class RemoteTable(Table):
def add_columns(
self,
transforms: Dict[str, str] | None = None,
transforms: Dict[str, str | FunctionApplication]
| FunctionApplication
| None = None,
*,
computed: Dict[str, str] | None = None,
) -> AddColumnsResult:
+54 -9
View File
@@ -72,6 +72,7 @@ from .index import (
FTS,
)
from .expr import Expr
from .functions import FunctionApplication
from .merge import LanceMergeInsertBuilder
from .pydantic import LanceModel, model_to_dict
from .query import (
@@ -1942,7 +1943,8 @@ class Table(ABC):
@abstractmethod
def add_columns(
self,
transforms: Dict[str, str]
transforms: Dict[str, str | FunctionApplication]
| FunctionApplication
| pa.Field
| List[pa.Field]
| pa.Schema
@@ -1955,13 +1957,21 @@ class Table(ABC):
Parameters
----------
transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema
transforms: Dict[str, str | FunctionApplication], FunctionApplication,
pa.Field, List[pa.Field], pa.Schema
A map of column name to a SQL expression to use to calculate the
value of the new column. These expressions will be evaluated for
each row in the table, and can reference existing columns.
Alternatively, a pyarrow Field or Schema can be provided to add
new columns with the specified data types. The new columns will
be initialized with null values.
A mapping with one ``FunctionApplication`` value keeps its scalar
or named-struct result in the named table column. A bare
named-struct application expands its ordered result fields as one
atomic sibling group; aliases come from ``rename(columns=...)``.
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str], optional
A map of column name to a SQL expression defining the column. The
column's type and inputs are derived from the expression, so no
@@ -4056,9 +4066,10 @@ class LanceTable(Table):
def add_columns(
self,
transforms: Dict[str, str]
| pa.field
| List[pa.field]
transforms: Dict[str, str | FunctionApplication]
| FunctionApplication
| pa.Field
| List[pa.Field]
| pa.Schema
| None = None,
*,
@@ -5992,9 +6003,10 @@ class AsyncTable:
async def add_columns(
self,
transforms: dict[str, str]
| pa.field
| List[pa.field]
transforms: dict[str, str | FunctionApplication]
| FunctionApplication
| pa.Field
| List[pa.Field]
| pa.Schema
| None = None,
*,
@@ -6005,12 +6017,19 @@ class AsyncTable:
Parameters
----------
transforms: Dict[str, str]
transforms: Dict[str, str | FunctionApplication] or FunctionApplication
A map of column name to a SQL expression to use to calculate the
value of the new column. These expressions will be evaluated for
each row in the table, and can reference existing columns.
Alternatively, you can pass a pyarrow field or schema to add
new columns with NULLs.
A mapping with one ``FunctionApplication`` value keeps its scalar
or named-struct result in the named table column. A bare
named-struct application expands its ordered result fields as one
atomic sibling group; aliases come from ``rename(columns=...)``.
Function columns are supported only on LanceDB Cloud and
Enterprise.
computed: Dict[str, str], optional
A map of column name to a SQL expression defining the column. The
column's type and inputs are derived from the expression.
@@ -6034,6 +6053,32 @@ class AsyncTable:
version: the new version number of the table after adding columns.
"""
function_application = None
function_output_name = None
if isinstance(transforms, FunctionApplication):
function_application = transforms
elif isinstance(transforms, dict) and any(
isinstance(value, FunctionApplication) for value in transforms.values()
):
if len(transforms) != 1 or not all(
isinstance(value, FunctionApplication) for value in transforms.values()
):
raise ValueError(
"one add_columns call declares exactly one Function sibling group"
)
function_output_name, function_application = next(iter(transforms.items()))
if function_application is not None:
if computed:
raise ValueError(
"add_columns cannot mix a Function application with SQL "
"computed columns"
)
function_application._ensure_declarable()
return await self._inner.add_function_columns(
function_application.to_canonical_json(), function_output_name
)
if isinstance(transforms, pa.Field):
transforms = [transforms]
if isinstance(transforms, list) and all(
@@ -14,6 +14,7 @@ from lancedb.functions import (
PythonRuntimeSpec,
RefreshColumnResult,
)
from lancedb.table import AsyncTable
FIXTURES = (
@@ -166,6 +167,8 @@ def test_binding_and_refresh_result_keep_stable_remote_fields():
assert binding.revision == 3
assert binding.function.version == "fv_01K3TEXT"
assert [output.output_ordinal for output in binding.outputs] == [0, 1]
assert binding.input_schema is not None
assert binding.output_schema is not None
result = RefreshColumnResult.from_json(
json.dumps(job_result("remote_refresh_job.json"))
@@ -223,3 +226,86 @@ def test_canonical_client_values_contain_secret_names_only():
canonical = json.loads(version.to_canonical_json())
assert canonical["required_secrets"] == ["HF_TOKEN"]
assert_no_secret_values(canonical)
class _FunctionDeclarationInner:
def __init__(self):
self.calls = []
async def add_function_columns(self, application_json, output_name):
self.calls.append((json.loads(application_json), output_name))
return "declared"
def known_application() -> FunctionApplication:
value = json.loads(fixture("remote_function_application.json"))
value.pop("future_application")
return FunctionApplication(**value)
@pytest.mark.asyncio
async def test_add_columns_routes_struct_as_one_and_grouped_expansion_atomically():
inner = _FunctionDeclarationInner()
table = AsyncTable(inner)
application = known_application()
result = await table.add_columns(
{"features": application._copy(update={"columns": {}})}
)
assert result == "declared"
assert inner.calls[-1][1] == "features"
bare = application._copy(update={"columns": {}}).rename(
columns={"normalized_text": "search_text"}
)
result = await table.add_columns(bare)
assert result == "declared"
assert inner.calls[-1][1] is None
assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
@pytest.mark.asyncio
async def test_add_columns_rejects_mixed_groups_and_unknown_newer_application():
inner = _FunctionDeclarationInner()
table = AsyncTable(inner)
application = known_application()
with pytest.raises(ValueError, match="exactly one Function sibling group"):
await table.add_columns({"a": application, "b": application})
future = json.loads(fixture("remote_function_application.json"))
application = FunctionApplication(**future)
with pytest.raises(ValueError, match="newer contract"):
await table.add_columns(application)
future.pop("future_application")
future["output"]["assignment"] = "cell_flag"
application = FunctionApplication(**future)
assert "assignment" not in json.loads(application.to_canonical_json())["output"]
with pytest.raises(ValueError, match="output.assignment"):
await table.add_columns(application)
assert inner.calls == []
def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable():
scalar = FunctionApplication.from_json(
json.dumps(
{
"function": {"name": "embed", "version": "fv_exact"},
"inputs": [],
"output": {
"kind": "scalar",
"arrow_type": "list<float32>",
"nullable": False,
},
"group_id": "fg_scalar",
}
)
)
with pytest.raises(ValueError, match="named-struct"):
scalar.rename(columns={"value": "embedding"})
application = known_application()._copy(update={"columns": {}})
renamed = application.rename(columns={"normalized_text": "search_text"})
assert dict(application.columns) == {}
assert dict(renamed.columns) == {"normalized_text": "search_text"}
+18
View File
@@ -1551,6 +1551,24 @@ impl Table {
})
}
pub fn add_function_columns(
self_: PyRef<'_, Self>,
application_json: String,
output_name: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let application =
lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let builder = match output_name {
Some(name) => inner.add_columns().function_as(name, application),
None => inner.add_columns().function(application),
};
let result = builder.execute().await.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
pub fn refresh_column(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
+68 -1
View File
@@ -74,6 +74,46 @@ fn validate_literal(value: &Value) -> Result<()> {
}
}
fn has_unknown_keys(value: &Value, allowed: &[&str]) -> bool {
value
.as_object()
.is_some_and(|object| object.keys().any(|key| !allowed.contains(&key.as_str())))
}
fn application_has_unknown_nested_fields(value: &Value) -> bool {
let Some(application) = value.as_object() else {
return false;
};
if application
.get("function")
.is_some_and(|value| has_unknown_keys(value, &["name", "version"]))
{
return true;
}
if application
.get("inputs")
.and_then(Value::as_array)
.is_some_and(|inputs| {
inputs
.iter()
.any(|input| has_unknown_keys(input, &["parameter", "kind", "value"]))
})
{
return true;
}
application.get("output").is_some_and(|output| {
has_unknown_keys(output, &["kind", "arrow_type", "nullable", "fields"])
|| output
.get("fields")
.and_then(Value::as_array)
.is_some_and(|fields| {
fields
.iter()
.any(|field| has_unknown_keys(field, &["name", "arrow_type", "nullable"]))
})
})
}
macro_rules! impl_json {
($type:ty) => {
impl $type {
@@ -358,6 +398,10 @@ pub struct FunctionApplication {
group_id: String,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
columns: BTreeMap<String, String>,
#[serde(default, flatten, skip_serializing)]
unknown_fields: BTreeMap<String, Value>,
#[serde(default, skip)]
unknown_nested_fields: bool,
}
impl FunctionApplication {
@@ -380,10 +424,18 @@ impl FunctionApplication {
pub fn columns(&self) -> &BTreeMap<String, String> {
&self.columns
}
/// Whether a newer writer attached application fields this client cannot
/// validate. Such applications remain readable but must not be declared.
pub fn has_unknown_fields(&self) -> bool {
!self.unknown_fields.is_empty() || self.unknown_nested_fields
}
/// Decode a remote application after validating the Slice 1 literal domain.
pub fn from_json(json: &str) -> Result<Self> {
let application: Self = from_json(json)?;
let value: Value = from_json(json)?;
let has_unknown_nested_fields = application_has_unknown_nested_fields(&value);
let mut application: Self = serde_json::from_value(value).map_err(invalid_json)?;
application.unknown_nested_fields = has_unknown_nested_fields;
application
.inputs
.iter()
@@ -433,6 +485,13 @@ pub struct FunctionBinding {
group_id: String,
inputs: Vec<InputBinding>,
outputs: Vec<OutputMapping>,
/// Exact Arrow schema presented to the Function, encoded with the Lance
/// Namespace Arrow JSON representation.
#[serde(default, skip_serializing_if = "Option::is_none")]
input_schema: Option<Value>,
/// Exact physical Arrow schema of the grouped table outputs.
#[serde(default, skip_serializing_if = "Option::is_none")]
output_schema: Option<Value>,
}
impl FunctionBinding {
@@ -459,6 +518,14 @@ impl FunctionBinding {
pub fn outputs(&self) -> &[OutputMapping] {
&self.outputs
}
pub fn input_schema(&self) -> Option<&Value> {
self.input_schema.as_ref()
}
pub fn output_schema(&self) -> Option<&Value> {
self.output_schema.as_ref()
}
}
impl_json!(FunctionBinding);
+224 -20
View File
@@ -2147,6 +2147,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
self.check_mutable().await?;
let table_schema = self.schema().await?;
crate::table::computed_columns::ensure_supported_function_metadata(table_schema.as_ref())?;
let table_def = TableDefinition::try_from_rich_schema(table_schema.clone())?;
let num_partitions = if self.server_version.support_multipart_write() {
@@ -2698,6 +2699,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
_read_columns: Option<Vec<String>>,
) -> Result<AddColumnsResult> {
self.check_mutable().await?;
crate::table::computed_columns::ensure_no_function_bindings_for_mutation(
self.schema().await?.as_ref(),
"schema evolution",
)?;
match transforms {
NewColumnTransform::SqlExpressions(expressions) => {
let body = expressions
@@ -2746,6 +2751,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result<AddColumnsResult> {
self.check_mutable().await?;
crate::table::computed_columns::ensure_no_function_bindings_for_mutation(
self.schema().await?.as_ref(),
"schema evolution",
)?;
// The server plans the declaration: expression validation, type
// inference and the persisted binding all happen there.
let entries = columns
@@ -2785,6 +2794,63 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
Ok(result)
}
async fn add_function_columns(
&self,
application: &crate::function::FunctionApplication,
output_name: Option<&str>,
) -> Result<AddColumnsResult> {
self.check_mutable().await?;
let schema = self.schema().await?;
let plan = crate::table::computed_columns::plan_function_application(
schema.as_ref(),
application,
output_name,
)?;
let new_columns = plan
.outputs
.iter()
.map(|output| {
serde_json::json!({
"name": output.output_name,
"all_null": true,
})
})
.collect::<Vec<_>>();
let mut body = serde_json::json!({
"new_columns": new_columns,
"function": {
"application": plan.application,
"binding_metadata_version": plan.binding_metadata_version,
"input_bindings": plan.input_bindings,
"input_schema": plan.input_schema,
"output_schema": plan.output_schema,
"outputs": plan.outputs,
},
});
self.apply_branch_body(&mut body);
let request = self
.client
.post(&format!("/v1/table/{}/add_columns/", self.identifier))
.json(&body);
let (request_id, response) = self.send(request, true).await?;
let response = self.check_table_response(&request_id, response).await?;
let body = response.text().await.err_to_http(request_id.clone())?;
if body.trim().is_empty() {
return Ok(AddColumnsResult { version: 0 });
}
let result: AddColumnsResult = serde_json::from_str(&body).map_err(|e| Error::Http {
source: format!("Failed to parse add Function columns response: {e}").into(),
request_id,
status_code: None,
})?;
self.invalidate_schema_cache();
self.track_write_version(result.version);
Ok(result)
}
async fn refresh_column(&self, _column: &str) -> Result<RefreshColumnResult> {
// The server runs a refresh as a job and does not report a fill
// count, so the blocking form has no honest result to return.
@@ -3810,11 +3876,14 @@ mod tests {
assert_eq!(rename, "y");
if old_server {
http::Response::builder().status(200).body("{}").unwrap()
http::Response::builder()
.status(200)
.body("{}".to_string())
.unwrap()
} else {
http::Response::builder()
.status(200)
.body(r#"{"version": 43}"#)
.body(r#"{"version": 43}"#.to_string())
.unwrap()
}
} else {
@@ -3945,11 +4014,14 @@ mod tests {
assert_eq!(predicate, "id in (1, 2, 3)");
if old_server {
http::Response::builder().status(200).body("{}").unwrap()
http::Response::builder()
.status(200)
.body("{}".to_string())
.unwrap()
} else {
http::Response::builder()
.status(200)
.body(r#"{"version": 43}"#)
.body(r#"{"version": 43}"#.to_string())
.unwrap()
}
} else {
@@ -6516,7 +6588,9 @@ mod tests {
#[tokio::test]
async fn test_add_columns(#[case] old_server: bool) {
let table = Table::new_with_handler("my_table", move |request| {
if request.url().path() == "/v1/table/my_table/add_columns/" {
if request.url().path() == "/v1/table/my_table/describe/" {
simple_describe_response()
} else if request.url().path() == "/v1/table/my_table/add_columns/" {
assert_eq!(request.method(), "POST");
assert_eq!(
request.headers().get("Content-Type").unwrap(),
@@ -6540,11 +6614,14 @@ mod tests {
assert_eq!(expression, "cast(NULL as int32)");
if old_server {
http::Response::builder().status(200).body("{}").unwrap()
http::Response::builder()
.status(200)
.body("{}".to_string())
.unwrap()
} else {
http::Response::builder()
.status(200)
.body(r#"{"version": 43}"#)
.body(r#"{"version": 43}"#.to_string())
.unwrap()
}
} else {
@@ -6569,19 +6646,22 @@ mod tests {
/// plan; the client never types the expression itself.
#[tokio::test]
async fn test_add_computed_columns_sends_the_expression() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
assert_eq!(request.url().path(), "/v1/table/my_table/add_columns/");
let body = request.body().unwrap().as_bytes().unwrap();
let value: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(
value["new_columns"],
serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
);
http::Response::builder()
.status(200)
.body(r#"{"version": 7}"#)
.unwrap()
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
"/v1/table/my_table/describe/" => simple_describe_response(),
"/v1/table/my_table/add_columns/" => {
assert_eq!(request.method(), "POST");
let body = request.body().unwrap().as_bytes().unwrap();
let value: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(
value["new_columns"],
serde_json::json!([{"name": "doubled", "computed": "x * 2"}])
);
http::Response::builder()
.status(200)
.body(r#"{"version": 7}"#.to_string())
.unwrap()
}
path => panic!("Unexpected path: {path}"),
});
let result = table
@@ -6593,6 +6673,129 @@ mod tests {
assert_eq!(result.version, 7);
}
#[tokio::test]
async fn test_add_scalar_function_column_sends_atomic_null_declaration() {
let table = Table::new_with_handler("my_table", |request| {
match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(
r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#,
)
.unwrap(),
"/v1/table/my_table/add_columns/" => {
let actual: serde_json::Value = serde_json::from_slice(
request.body().unwrap().as_bytes().unwrap(),
)
.unwrap();
let expected: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json"
))
.unwrap();
assert_eq!(actual, expected);
http::Response::builder()
.status(200)
.body(r#"{"version":8}"#)
.unwrap()
}
path => panic!("Unexpected path: {path}"),
}
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"embed","version":"fv_01K3EXACT"},
"inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}],
"output":{"kind":"scalar","arrow_type":"list<float32>","nullable":false},
"group_id":"fg_scalar"
}"#,
)
.unwrap();
let result = table
.add_columns()
.function_as("embedding", application)
.execute()
.await
.unwrap();
assert_eq!(result.version, 8);
}
#[tokio::test]
async fn test_add_named_struct_function_expands_one_atomic_sibling_group() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(
r#"{"version":1,"schema":{"fields":[
{"name":"title","nullable":true,"type":{"type":"string"}},
{"name":"body","nullable":true,"type":{"type":"string"}}
]}}"#,
)
.unwrap(),
"/v1/table/my_table/add_columns/" => {
let actual: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
let expected: serde_json::Value = serde_json::from_str(include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_grouped_declaration_request.json"
))
.unwrap();
assert_eq!(actual, expected);
http::Response::builder()
.status(200)
.body(r#"{"version":9}"#)
.unwrap()
}
path => panic!("Unexpected path: {path}"),
});
let application = crate::function::FunctionApplication::from_json(
r#"{
"function":{"name":"text_features","version":"fv_01K3TEXT"},
"inputs":[
{"parameter":"title","kind":"column","value":{"path":"title"}},
{"parameter":"body","kind":"column","value":{"path":"body"}}
],
"output":{"kind":"named_struct","fields":[
{"name":"normalized_text","arrow_type":"utf8","nullable":false},
{"name":"token_count","arrow_type":"int64","nullable":false}
]},
"group_id":"fg_01K3TEXT",
"columns":{"normalized_text":"search_text"}
}"#,
)
.unwrap();
let result = table
.add_columns()
.function(application)
.execute()
.await
.unwrap();
assert_eq!(result.version, 9);
}
#[tokio::test]
async fn test_add_columns_fails_closed_on_newer_function_binding_metadata() {
let table = Table::new_with_handler("my_table", |request| {
match request.url().path() {
"/v1/table/my_table/describe/" => http::Response::builder()
.status(200)
.body(
r#"{"version":1,"schema":{"fields":[{"name":"x","nullable":true,"type":{"type":"int32"}}],"metadata":{"lancedb::function_bindings":"{\"version\":2,\"bindings\":[]}"}}}"#,
)
.unwrap(),
path => panic!("mutation request must not be sent: {path}"),
}
});
let err = table
.add_columns()
.computed("doubled", "x * 2")
.execute()
.await
.unwrap_err();
assert!(matches!(err, Error::NotSupported { .. }));
}
/// A remote refresh is a server job: the async form returns its handle,
/// and the blocking form refuses rather than invent a fill count.
#[tokio::test]
@@ -10911,6 +11114,7 @@ mod tests {
.status(200)
.body("{}".to_string())
.unwrap(),
"/v1/table/my_table/describe/" => simple_describe_response(),
"/v1/table/my_table/add_columns/"
| "/v1/table/my_table/alter_columns/"
| "/v1/table/my_table/drop_columns/" => {
+11
View File
@@ -760,6 +760,16 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
message: "computed columns are not supported on this table type".into(),
})
}
/// Declare one immutable registered-Function output group.
async fn add_function_columns(
&self,
_application: &crate::function::FunctionApplication,
_output_name: Option<&str>,
) -> Result<AddColumnsResult> {
Err(Error::NotSupported {
message: "Function columns are supported only on LanceDB Cloud and Enterprise".into(),
})
}
/// Fill a computed column's unfilled rows.
///
/// The default returns `NotSupported`; Lance-backed tables override it.
@@ -3158,6 +3168,7 @@ impl BaseTable for NativeTable {
let ds = self.dataset.get().await?;
let table_schema = Schema::from(&ds.schema().clone());
computed_columns::ensure_supported_function_metadata(&table_schema)?;
computed_columns::ensure_not_written(
&table_schema,
add.data.schema().fields().iter().map(|f| f.name().as_str()),
+71 -11
View File
@@ -9,6 +9,7 @@ use lance::dataset::NewColumnTransform;
use super::BaseTable;
use super::schema_evolution::AddColumnsResult;
use crate::function::FunctionApplication;
use crate::{Error, Result};
/// Adds columns to a table. See [`Table::add_columns`](super::Table::add_columns).
@@ -16,6 +17,7 @@ pub struct AddColumnsBuilder {
parent: Arc<dyn BaseTable>,
transform: Option<NewColumnTransform>,
computed: Vec<(String, String)>,
function: Option<(FunctionApplication, Option<String>)>,
read_columns: Option<Vec<String>>,
}
@@ -25,6 +27,7 @@ impl std::fmt::Debug for AddColumnsBuilder {
.field("parent", &self.parent)
.field("has_transform", &self.transform.is_some())
.field("computed", &self.computed)
.field("has_function", &self.function.is_some())
.field("read_columns", &self.read_columns)
.finish()
}
@@ -36,6 +39,7 @@ impl AddColumnsBuilder {
parent,
transform: None,
computed: Vec::new(),
function: None,
read_columns: None,
}
}
@@ -83,6 +87,48 @@ impl AddColumnsBuilder {
self
}
/// Declare every field of a named-struct Function result as one atomic
/// sibling group. Result-field aliases come from
/// [`FunctionApplication::columns`](crate::function::FunctionApplication::columns).
///
/// ```
/// # use lancedb::Table;
/// # use lancedb::function::FunctionApplication;
/// # async fn declare(table: &Table, application: FunctionApplication) -> lancedb::Result<()> {
/// table.add_columns().function(application).execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn function(mut self, application: FunctionApplication) -> Self {
self.function = Some((application, None));
self
}
/// Declare a scalar or entire named-struct Function result as one table
/// column. The physical column starts all-null and is materialized by the
/// remote Function refresh path.
///
/// ```
/// # use lancedb::Table;
/// # use lancedb::function::FunctionApplication;
/// # async fn declare(table: &Table, application: FunctionApplication) -> lancedb::Result<()> {
/// table
/// .add_columns()
/// .function_as("embedding", application)
/// .execute()
/// .await?;
/// # Ok(())
/// # }
/// ```
pub fn function_as(
mut self,
name: impl Into<String>,
application: FunctionApplication,
) -> Self {
self.function = Some((application, Some(name.into())));
self
}
/// Limit which existing columns a [`NewColumnTransform::BatchUDF`] mapper
/// receives. Every other transform, and a computed column, determines what
/// it reads, so setting this alongside one is an error rather than a silent
@@ -98,21 +144,23 @@ impl AddColumnsBuilder {
parent,
transform,
computed,
function,
read_columns,
} = self;
match (transform, computed.is_empty()) {
(None, true) => Err(Error::InvalidInput {
let declaration_count = usize::from(!computed.is_empty()) + usize::from(function.is_some());
if transform.is_some() && declaration_count != 0 || declaration_count > 1 {
return Err(Error::InvalidInput {
message: "add_columns cannot mix transforms, SQL computed columns, and a Function application; they cannot be added atomically in one call"
.into(),
});
}
match (transform, computed.is_empty(), function) {
(None, true, None) => Err(Error::InvalidInput {
message: "add_columns requires a transform or a computed column".into(),
}),
// The two commit through different transforms, so one call covering
// both would be two commits and could half-apply.
(Some(_), false) => Err(Error::InvalidInput {
message: "add_columns cannot mix a transform with computed columns; \
they cannot be added atomically in one call"
.into(),
}),
(Some(transform), true) => {
(Some(transform), true, None) => {
if read_columns.is_some() && !matches!(transform, NewColumnTransform::BatchUDF(_)) {
return Err(Error::InvalidInput {
message: "read_columns applies only to a BatchUDF transform; \
@@ -122,7 +170,7 @@ impl AddColumnsBuilder {
}
parent.add_columns(transform, read_columns).await
}
(None, false) => {
(None, false, None) => {
if read_columns.is_some() {
return Err(Error::InvalidInput {
message: "read_columns applies only to a BatchUDF transform; \
@@ -132,6 +180,18 @@ impl AddColumnsBuilder {
}
parent.add_computed_columns(&computed).await
}
(None, true, Some((application, output_name))) => {
if read_columns.is_some() {
return Err(Error::InvalidInput {
message: "read_columns does not apply to a Function application; its inputs are already bound"
.into(),
});
}
parent
.add_function_columns(&application, output_name.as_deref())
.await
}
_ => unreachable!("mixed add_columns modes were rejected above"),
}
}
}
File diff suppressed because it is too large Load Diff
+4
View File
@@ -233,6 +233,10 @@ pub(crate) async fn execute_merge_insert(
params: MergeInsertBuilder,
new_data: Box<dyn RecordBatchReader + Send>,
) -> Result<MergeResult> {
super::computed_columns::ensure_no_function_bindings_for_mutation(
table.schema().await?.as_ref(),
"merge_insert",
)?;
match lsm::lsm_dispatch_decision(table, &params).await? {
lsm::LsmDispatch::Lsm(plan) => {
let future =
+3
View File
@@ -169,6 +169,9 @@ fn declared_expression(dataset: &Dataset, column: &str) -> Result<String> {
})?;
match declaration.kind {
ComputedColumnKind::Sql { expression } => Ok(expression),
ComputedColumnKind::Function { .. } => Err(Error::NotSupported {
message: "registered Function columns are refreshed only by a remote server Job".into(),
}),
ComputedColumnKind::Unrecognized { kind } => Err(Error::NotSupported {
message: format!(
"computed column '{column}' is defined by '{kind}', which this version of \
@@ -101,6 +101,10 @@ pub(crate) async fn execute_add_columns(
transforms: NewColumnTransform,
read_columns: Option<Vec<String>>,
) -> Result<AddColumnsResult> {
computed_columns::ensure_no_function_bindings_for_mutation(
table.schema().await?.as_ref(),
"schema evolution",
)?;
// Declarations are admitted only through [`execute_declare`].
match &transforms {
NewColumnTransform::AllNulls(schema) => {
@@ -124,6 +128,10 @@ pub(crate) async fn execute_declare(
// checked against latest committed state, not this handle's snapshot.
// The catch-up flag outlives unset and marks retained SSTable rows.
table.checkout_latest().await?;
computed_columns::ensure_no_function_bindings_for_mutation(
table.schema().await?.as_ref(),
"schema evolution",
)?;
let catchup = table.dataset.get().await?.manifest().reader_feature_flags
& lance_table::feature_flags::FLAG_MEM_WAL_INDEX_CATCHUP
!= 0;
@@ -163,6 +171,10 @@ pub(crate) async fn execute_alter_columns(
// Nullability is not part of what an expression resolves against, so only
// a rename or a retype can invalidate a binding.
let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema()));
computed_columns::ensure_no_function_bindings_for_mutation(
schema.as_ref(),
"schema evolution",
)?;
let rebinding = alterations
.iter()
.filter(|alteration| alteration.rename.is_some() || alteration.data_type.is_some())
@@ -190,6 +202,10 @@ pub(crate) async fn execute_drop_columns(
) -> Result<DropColumnsResult> {
table.dataset.ensure_mutable()?;
let mut dataset = (*table.dataset.get().await?).clone();
computed_columns::ensure_no_function_bindings_for_mutation(
&ArrowSchema::from(dataset.schema()),
"schema evolution",
)?;
computed_columns::ensure_not_an_input(
&std::sync::Arc::new(ArrowSchema::from(dataset.schema())),
columns,
@@ -215,6 +231,7 @@ pub(crate) async fn execute_update_field_metadata(
// binding out from under a refresh. A replace on a declared column would
// silently erase it.
let schema = ArrowSchema::from(dataset.schema());
computed_columns::ensure_no_function_bindings_for_mutation(&schema, "schema evolution")?;
let declared: Vec<String> = computed_columns::computed_columns(&schema)
.into_iter()
.map(|declaration| declaration.name)
+4
View File
@@ -82,6 +82,10 @@ pub(crate) async fn execute_update(
// 1. Snapshot the current dataset
let dataset = table.dataset.get().await?;
super::computed_columns::ensure_no_function_bindings_for_mutation(
&arrow_schema::Schema::from(dataset.schema()),
"update",
)?;
super::computed_columns::ensure_not_written(
&arrow_schema::Schema::from(dataset.schema()),
update.columns.iter().map(|(name, _)| name.as_str()),
@@ -88,6 +88,8 @@ fn application_and_binding_match_shared_remote_goldens() {
assert_eq!(binding.function().version, "fv_01K3TEXT");
assert_eq!(binding.outputs()[0].output_ordinal, 0);
assert_eq!(binding.outputs()[1].output_ordinal, 1);
assert!(binding.input_schema().is_some());
assert!(binding.output_schema().is_some());
assert_eq!(
binding.to_canonical_json().expect("canonical JSON"),
fixture("remote_function_binding.canonical.json").trim()
@@ -1 +1 @@
{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}],"revision":3}
{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"group_id":"fg_01K3TEXT","input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}],"revision":3}
@@ -11,5 +11,17 @@
{"result_field": "normalized_text", "output_name": "search_text", "output_field_id": 21, "output_ordinal": 0, "arrow_type": "utf8", "nullable": false},
{"result_field": "token_count", "output_name": "search_token_count", "output_field_id": 22, "output_ordinal": 1, "arrow_type": "int64", "nullable": false}
],
"input_schema": {
"fields": [
{"name": "title", "nullable": true, "type": {"type": "utf8"}},
{"name": "body", "nullable": true, "type": {"type": "utf8"}}
]
},
"output_schema": {
"fields": [
{"name": "search_text", "nullable": true, "type": {"type": "utf8"}},
{"name": "search_token_count", "nullable": true, "type": {"type": "int64"}}
]
},
"future_binding": {"metadata_revision": 1}
}
@@ -0,0 +1,45 @@
{
"new_columns": [
{"name": "search_text", "all_null": true},
{"name": "token_count", "all_null": true}
],
"function": {
"application": {
"function": {"name": "text_features", "version": "fv_01K3TEXT"},
"inputs": [
{"parameter": "title", "kind": "column", "value": {"path": "title"}},
{"parameter": "body", "kind": "column", "value": {"path": "body"}}
],
"output": {
"kind": "named_struct",
"fields": [
{"name": "normalized_text", "arrow_type": "utf8", "nullable": false},
{"name": "token_count", "arrow_type": "int64", "nullable": false}
]
},
"group_id": "fg_01K3TEXT",
"columns": {"normalized_text": "search_text"}
},
"binding_metadata_version": 1,
"input_bindings": [
{"parameter": "title", "field_path": "title", "arrow_type": "utf8", "nullable": true},
{"parameter": "body", "field_path": "body", "arrow_type": "utf8", "nullable": true}
],
"input_schema": {
"fields": [
{"name": "title", "nullable": true, "type": {"type": "utf8"}},
{"name": "body", "nullable": true, "type": {"type": "utf8"}}
]
},
"output_schema": {
"fields": [
{"name": "search_text", "nullable": true, "type": {"type": "utf8"}},
{"name": "token_count", "nullable": true, "type": {"type": "int64"}}
]
},
"outputs": [
{"result_field": "normalized_text", "output_name": "search_text", "output_ordinal": 0},
{"result_field": "token_count", "output_name": "token_count", "output_ordinal": 1}
]
}
}
@@ -0,0 +1,41 @@
{
"new_columns": [
{"name": "embedding", "all_null": true}
],
"function": {
"application": {
"function": {"name": "embed", "version": "fv_01K3EXACT"},
"inputs": [
{"parameter": "text", "kind": "column", "value": {"path": "description"}}
],
"output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false},
"group_id": "fg_scalar"
},
"binding_metadata_version": 1,
"input_bindings": [
{"parameter": "text", "field_path": "description", "arrow_type": "utf8", "nullable": true}
],
"input_schema": {
"fields": [
{"name": "text", "nullable": true, "type": {"type": "utf8"}}
]
},
"output_schema": {
"fields": [
{
"name": "embedding",
"nullable": true,
"type": {
"type": "list",
"fields": [
{"name": "item", "nullable": false, "type": {"type": "float32"}}
]
}
}
]
},
"outputs": [
{"result_field": "$value", "output_name": "embedding", "output_ordinal": 0}
]
}
}