Compare commits

..

1 Commits

Author SHA1 Message Date
Lance Release cad46fc683 Bump version: 0.38.0-beta.13 → 0.38.0-beta.14 2026-08-30 15:16:00 +00:00
31 changed files with 96 additions and 901 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0"
current_version = "0.38.0-beta.14"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Generated
+27 -29
View File
@@ -535,9 +535,9 @@ dependencies = [
[[package]]
name = "async-trait"
version = "0.1.92"
version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
@@ -1443,9 +1443,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
[[package]]
name = "bytemuck"
version = "1.25.2"
version = "1.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
dependencies = [
"bytemuck_derive",
]
@@ -5402,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0"
version = "0.38.0-beta.13"
dependencies = [
"ahash",
"anyhow",
@@ -5490,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0"
version = "0.38.0-beta.13"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5515,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0"
version = "0.38.0-beta.13"
dependencies = [
"arrow",
"async-trait",
@@ -5748,9 +5748,9 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.34"
version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "loom"
@@ -6001,9 +6001,9 @@ dependencies = [
[[package]]
name = "moka"
version = "0.12.16"
version = "0.12.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
dependencies = [
"async-lock",
"crossbeam-channel",
@@ -6097,15 +6097,14 @@ dependencies = [
[[package]]
name = "napi"
version = "3.12.2"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
dependencies = [
"bitflags 2.11.1",
"chrono",
"ctor 1.0.12",
"futures",
"libc",
"napi-build",
"napi-sys",
"nohash-hasher",
@@ -6117,15 +6116,15 @@ dependencies = [
[[package]]
name = "napi-build"
version = "2.4.1"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
[[package]]
name = "napi-derive"
version = "3.6.3"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
dependencies = [
"convert_case",
"ctor 1.0.12",
@@ -6137,9 +6136,9 @@ dependencies = [
[[package]]
name = "napi-derive-backend"
version = "6.1.2"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
dependencies = [
"convert_case",
"proc-macro2",
@@ -8602,9 +8601,9 @@ dependencies = [
[[package]]
name = "roaring"
version = "0.11.5"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
dependencies = [
"bytemuck",
"byteorder",
@@ -9064,9 +9063,9 @@ dependencies = [
[[package]]
name = "serde_with"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
dependencies = [
"base64 0.22.1",
"bs58",
@@ -9074,7 +9073,6 @@ dependencies = [
"hex",
"indexmap 1.9.3",
"indexmap 2.14.0",
"jiff",
"schemars 0.9.0",
"schemars 1.2.1",
"serde_core",
@@ -9085,9 +9083,9 @@ dependencies = [
[[package]]
name = "serde_with_macros"
version = "3.22.0"
version = "3.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
dependencies = [
"darling 0.23.0",
"proc-macro2",
@@ -10454,9 +10452,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
version = "1.26.0"
version = "1.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
dependencies = [
"getrandom 0.4.2",
"js-sys",
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0</version>
<version>0.38.0-beta.14</version>
</dependency>
```
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-final.0</version>
<version>0.38.0-beta.14</version>
<relativePath>../pom.xml</relativePath>
</parent>
+1 -1
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-final.0</version>
<version>0.38.0-beta.14</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0"
version = "0.38.0-beta.14"
publish = false
license.workspace = true
description.workspace = true
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0",
"version": "0.38.0-beta.14",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0",
"version": "0.38.0-beta.14",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
+1 -5
View File
@@ -664,11 +664,7 @@ impl JsFullTextQuery {
}
fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
// `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
// path, which is only usable from generated `#[napi]` argument conversion. This is a
// manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
// unwraps the class without requiring a borrow scope.
if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
Ok(FullTextSearchQuery::new_query(query.inner.clone()))
} else if let Ok(Some(query_text)) = query.get::<String>("query") {
let mut query_text = query_text;
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0"
version = "0.38.0-beta.14"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
-1
View File
@@ -150,7 +150,6 @@ class Connection(object):
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
async def cancel_job(self, job_id: str) -> bool: ...
-18
View File
@@ -712,16 +712,6 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog.
Returns True when the version changed to Dropped and False for an
idempotent replay. Local connections raise NotImplementedError.
"""
raise NotImplementedError(
"Function catalog operations are not supported for this connection type"
)
def job(self, job_id: str) -> Job:
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
@@ -1423,10 +1413,6 @@ class LanceDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
@@ -2257,10 +2243,6 @@ class AsyncConnection(object):
"""Open one exact immutable Function version from the remote catalog."""
return FunctionVersion.from_json(await self._inner.get_function(name, version))
async def drop_function(self, name: str, *, version: str) -> bool:
"""Drop one exact immutable Function version from the remote catalog."""
return await self._inner.drop_function(name, version)
async def list_jobs(self) -> List[JobInfo]:
"""List server-side jobs across the database's tables."""
return await self._inner.list_jobs()
+7 -87
View File
@@ -49,8 +49,6 @@ from pydantic import (
model_validator,
)
from .schema import is_blob_v2_field as _is_blob_v2_field
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
@@ -520,7 +518,6 @@ class RefreshColumnResult(_RemoteValue):
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
_GRAMMAR_PRIMITIVES = (
@@ -537,7 +534,6 @@ _GRAMMAR_PRIMITIVES = (
(pa.float32(), "float32"),
(pa.float64(), "float64"),
(pa.string(), "utf8"),
(pa.large_string(), "large_utf8"),
(pa.binary(), "binary"),
(pa.date32(), "date32"),
(pa.date64(), "date64"),
@@ -585,90 +581,20 @@ def _validate_exact_arrow_field(field: pa.Field) -> None:
"unsupported Arrow type for Function signature: field names "
"must not be empty"
)
if _is_blob_v2_field(field):
if not _has_supported_blob_v2_layout(field):
raise TypeError(
"unsupported Arrow type for Function signature: lance.blob.v2 "
f"requires a supported Blob storage layout, got {field}"
)
elif field.metadata:
if field.metadata:
raise TypeError(
"unsupported Arrow type for Function signature: field metadata "
f"is not supported, got {field}"
)
def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
data_type = field.type
if isinstance(data_type, pa.ExtensionType):
data_type = data_type.storage_type
if not pa.types.is_struct(data_type):
return False
fields = tuple(data_type)
def matches(spec, compare_nullable) -> bool:
return len(fields) == len(spec) and all(
actual.name == name
and actual.type == expected_type
and (not check_nullable or actual.nullable == nullable)
for actual, (name, expected_type, nullable), check_nullable in zip(
fields, spec, compare_nullable
)
)
logical_minimal = (
("data", pa.large_binary(), True),
("uri", pa.utf8(), True),
)
logical_full = logical_minimal + (
("position", pa.uint64(), True),
("size", pa.uint64(), True),
)
prepared = (
("kind", pa.uint8(), True),
("data", pa.large_binary(), True),
("uri", pa.utf8(), True),
("blob_id", pa.uint32(), True),
("blob_size", pa.uint64(), True),
("position", pa.uint64(), True),
)
descriptor = (
("kind", pa.uint8(), False),
("position", pa.uint64(), False),
("size", pa.uint64(), False),
("blob_id", pa.uint32(), False),
("blob_uri", pa.utf8(), False),
)
return (
matches(logical_minimal, (True, True))
or matches(logical_full, (True, True, False, False))
or matches(prepared, (True,) * len(prepared))
or matches(descriptor, (False,) * len(descriptor))
)
def _canonical_arrow_field(field: pa.Field) -> str:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
return _FUNCTION_BLOB_V2_TYPE
return _canonical_arrow_type(field.type)
def _exact_arrow_field(field: pa.Field) -> dict[str, Any]:
_validate_exact_arrow_field(field)
if _is_blob_v2_field(field):
raise TypeError(
"unsupported Arrow type for Function signature: nested Blob v2 "
"fields are not supported; declare Blob parameters or named result "
"fields directly"
)
value = {
return {
"name": field.name,
"nullable": field.nullable,
"type": _exact_arrow_type(field.type),
}
return value
def _exact_arrow_type(data_type: pa.DataType) -> dict[str, Any]:
@@ -792,11 +718,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
if output.metadata:
raise TypeError("Function output schema metadata is not supported")
fields = tuple(output)
elif (
isinstance(output, pa.Field)
and not _is_blob_v2_field(output)
and pa.types.is_struct(output.type)
):
elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
_validate_exact_arrow_field(output)
if output.nullable:
raise ValueError("Function output must be non-nullable")
@@ -818,7 +740,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
raise ValueError("Function output must be non-nullable")
return FunctionOutput(
kind="scalar",
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=False,
)
@@ -836,7 +758,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
fields=tuple(
FunctionResultField(
name=field.name,
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=False,
)
for field in fields
@@ -870,7 +792,7 @@ def _infer_signature(
inputs = tuple(
FunctionParameter(
name=field.name,
arrow_type=_canonical_arrow_field(field),
arrow_type=_canonical_arrow_type(field.type),
nullable=field.nullable,
)
for field in input_schema
@@ -893,9 +815,7 @@ def _infer_signature(
inputs.append(
FunctionParameter(
name=parameter.name,
arrow_type=_canonical_arrow_field(
pa.field(parameter.name, data_type, nullable=nullable)
),
arrow_type=_canonical_arrow_type(data_type),
nullable=nullable,
)
)
-4
View File
@@ -749,10 +749,6 @@ class RemoteDBConnection(DBConnection):
def get_function(self, name: str, *, version: str) -> FunctionVersion:
return LOOP.run(self._conn.get_function(name, version=version))
@override
def drop_function(self, name: str, *, version: str) -> bool:
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def list_jobs(self) -> List["JobInfo"]:
"""List server-side jobs across the database's tables."""
@@ -12,8 +12,6 @@ from datetime import date
import http.server
import json
from pathlib import Path
import subprocess
import sys
import threading
from typing import Optional
@@ -21,13 +19,7 @@ import pyarrow as pa
import pytest
import lancedb
from lancedb.functions import (
PythonRuntimeSpec,
UdfDefinition,
_canonical_arrow_type,
_GRAMMAR_PRIMITIVES,
udf,
)
from lancedb.functions import PythonRuntimeSpec, UdfDefinition, udf
THRESHOLD = 20
_CACHE = None
@@ -69,80 +61,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
}
def _main_udf_source(
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
) -> str:
return (
"from __future__ import annotations\n"
"from lancedb.functions import udf\n"
f"THRESHOLD = {threshold}\n"
"\n"
"@udf\n"
f"def label(value: {input_annotation}) -> str:\n"
f" return 'big' if value {comparison} THRESHOLD else 'small'\n"
"\n"
"assert label.__module__ == '__main__'\n"
"print(label.registration_request.to_canonical_json())\n"
)
def _run_main_udf(path: Path, source: str) -> dict:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(source)
result = subprocess.run(
[sys.executable, str(path)],
check=True,
capture_output=True,
text=True,
)
return json.loads(result.stdout)
def test_main_udf_registration_identity_is_stable_across_processes_and_paths(
tmp_path,
):
source = _main_udf_source()
original_path = tmp_path / "original" / "job.py"
moved_path = tmp_path / "moved" / "renamed_job.py"
original_runs = [_run_main_udf(original_path, source) for _ in range(2)]
moved_run = _run_main_udf(moved_path, source)
assert len({run["artifact"]["digest"] for run in [*original_runs, moved_run]}) == 1
assert all(
run["signature"] == original_runs[0]["signature"]
for run in [original_runs[1], moved_run]
)
assert original_runs[0] == original_runs[1] == moved_run
body_change = _run_main_udf(
tmp_path / "changes" / "body.py", _main_udf_source(comparison=">")
)
global_change = _run_main_udf(
tmp_path / "changes" / "global.py", _main_udf_source(threshold=21)
)
annotation_change = _run_main_udf(
tmp_path / "changes" / "annotation.py",
_main_udf_source(input_annotation="float"),
)
baseline = original_runs[0]
assert baseline["signature"] == body_change["signature"]
assert baseline["signature"] == global_change["signature"]
assert baseline["signature"] != annotation_change["signature"]
assert (
len(
{
baseline["artifact"]["digest"],
body_change["artifact"]["digest"],
global_change["artifact"]["digest"],
annotation_change["artifact"]["digest"],
}
)
== 4
)
def _run_packaged(definition, *args):
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
source = base64.b64decode(definition.registration_request.artifact.content.data)
@@ -303,6 +221,8 @@ def test_udf_resolves_module_globals_before_builtins(tmp_path):
def test_canonical_arrow_type_prefers_the_compact_grammar():
from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
golden = json.loads(
(
Path(__file__).parents[3]
@@ -323,6 +243,7 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
for outside in [
pa.timestamp("us"),
pa.decimal128(10, 2),
pa.large_string(),
pa.large_binary(),
pa.binary(4),
pa.duration("s"),
@@ -516,6 +437,8 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
from lancedb.functions import _canonical_arrow_type
nullable = pa.list_(pa.float32())
assert json.loads(_canonical_arrow_type(nullable)) == {
"type": "list",
@@ -605,7 +528,6 @@ def _arrow_type_from_golden(spec: dict) -> pa.DataType:
"null": pa.null(),
"bool": pa.bool_(),
"utf8": pa.string(),
"large_utf8": pa.large_string(),
"binary": pa.binary(),
"float16": pa.float16(),
"float32": pa.float32(),
@@ -622,6 +544,8 @@ def test_arrow_type_grammar_matches_the_shared_golden():
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
).read_text()
)
from lancedb.functions import _canonical_arrow_type
emitted = {
case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
for case in golden["valid"]
@@ -654,151 +578,6 @@ def test_explicit_arrow_schema_is_deterministic():
assert signature.output.nullable is False
def test_blob_fields_use_the_scalar_function_semantic_type():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=lancedb.blob("result", nullable=False),
)
def copy_blob(image):
return image
signature = copy_blob.registration_request.signature
assert signature.inputs[0].arrow_type == "blob_v2"
assert signature.output.kind == "scalar"
assert signature.output.arrow_type == "blob_v2"
def test_named_struct_function_can_include_a_blob_result_field():
@udf(
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
output_schema=pa.schema(
[
lancedb.blob("thumbnail", nullable=False),
pa.field("width", pa.int32(), nullable=False),
]
),
)
def inspect_blob(image):
return {"thumbnail": image, "width": 1}
output = inspect_blob.registration_request.signature.output
assert output.kind == "named_struct"
assert [(field.name, field.arrow_type) for field in output.fields] == [
("thumbnail", "blob_v2"),
("width", "int32"),
]
def test_metadata_marked_blob_field_uses_the_semantic_type():
extension = lancedb.blob("image", nullable=False).type
storage = (
extension.storage_type if isinstance(extension, pa.ExtensionType) else extension
)
metadata_blob = pa.field(
"image",
storage,
nullable=False,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
@udf(
input_schema=pa.schema([metadata_blob]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(image):
return len(image)
assert blob_size.registration_request.signature.inputs[0].arrow_type == "blob_v2"
def test_blob_marker_rejects_invalid_storage_layout():
malformed = pa.field(
"image",
pa.int64(),
nullable=False,
metadata={"ARROW:extension:name": "lance.blob.v2"},
)
with pytest.raises(TypeError, match="requires a supported Blob storage layout"):
@udf(
input_schema=pa.schema([malformed]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(image):
return len(image)
def test_nested_blob_signature_field_has_a_clear_error():
nested = pa.field(
"value",
pa.struct([lancedb.blob("image", nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="nested Blob v2 fields are not supported"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("size", pa.int64(), nullable=False),
)
def blob_size(value):
return len(value["image"])
def test_nested_non_blob_extension_is_not_silently_unwrapped():
class TestExtension(pa.ExtensionType):
def __init__(self):
super().__init__(pa.int64(), "test.function.extension")
def __arrow_ext_serialize__(self):
return b""
@classmethod
def __arrow_ext_deserialize__(cls, storage_type, serialized):
return cls()
nested = pa.field(
"value",
pa.struct([pa.field("extended", TestExtension(), nullable=False)]),
nullable=False,
)
with pytest.raises(TypeError, match="unsupported Arrow type"):
@udf(
input_schema=pa.schema([nested]),
output_schema=pa.field("result", pa.int64(), nullable=False),
)
def extension_value(value):
return value["extended"]
def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
input_schema = pa.schema([pa.field("text", pa.large_string(), nullable=True)])
output_schema = pa.field("result", pa.large_string(), nullable=False)
@udf(input_schema=input_schema, output_schema=output_schema)
def preserve(text):
return text
signature = preserve.registration_request.signature
assert signature.inputs[0].arrow_type == "large_utf8"
assert signature.inputs[0].nullable is True
assert signature.output.arrow_type == "large_utf8"
assert signature.output.nullable is False
nested = pa.struct([pa.field("text", pa.large_string(), nullable=True)])
assert json.loads(_canonical_arrow_type(nested)) == {
"type": "struct",
"fields": [
{
"name": "text",
"nullable": True,
"type": {"type": "large_utf8"},
}
],
}
def test_nested_struct_output_uses_canonical_exact_json():
token = pa.struct(
[
@@ -1017,8 +796,6 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
db.create_function_async(normalize_score)
with pytest.raises(NotImplementedError, match=message):
db.get_function("normalize_score", version="fv_exact")
with pytest.raises(NotImplementedError, match=message):
db.drop_function("normalize_score", version="fv_exact")
@contextlib.contextmanager
@@ -1064,12 +841,6 @@ def _mock_remote_function_catalog():
"version": "fv_exact",
}
response = state["version"]
elif self.path == "/v1/functions/drop":
assert body == {
"name": "normalize_score",
"version": "fv_exact",
}
response = {"dropped": True}
else:
status = 404
response = {"error": "not found"}
@@ -1128,40 +899,3 @@ def test_blocking_remote_registration_returns_function_version():
"/v1/functions/create",
"/v1/jobs/describe",
]
def test_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
@pytest.mark.asyncio
async def test_async_remote_drop_function_sends_exact_version():
with _mock_remote_function_catalog() as (host, state):
db = await lancedb.connect_async(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert await db.drop_function("normalize_score", version="fv_exact") is True
assert state["requests"] == [
(
"/v1/functions/drop",
{"name": "normalize_score", "version": "fv_exact"},
)
]
-11
View File
@@ -629,17 +629,6 @@ impl Connection {
})
}
pub fn drop_function(
self_: PyRef<'_, Self>,
name: String,
version: String,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_function(name, version).await.infer_error()
})
}
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.38.0"
version = "0.38.0-beta.14"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
-15
View File
@@ -523,21 +523,6 @@ impl Connection {
.await
}
/// Drop one exact immutable Function version from the remote catalog.
///
/// Returns `true` when the server appended a Dropped transition and
/// `false` for an idempotent replay. Local databases return
/// [`Error::NotSupported`].
pub async fn drop_function(
&self,
name: impl AsRef<str>,
version: impl AsRef<str>,
) -> Result<bool> {
self.internal
.drop_function(name.as_ref(), version.as_ref())
.await
}
/// Rename a table in the database.
///
/// This is only supported in LanceDB Cloud.
-4
View File
@@ -307,10 +307,6 @@ pub trait Database:
) -> Result<crate::function::FunctionVersion> {
function_catalog_not_supported()
}
/// Drop one exact immutable Function version from the remote catalog.
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
function_catalog_not_supported()
}
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
/// waiting on or cancelling the job. The handle is constructed without a
/// server round trip; an unknown id surfaces when the handle is used.
+3 -10
View File
@@ -539,7 +539,9 @@ impl Database for LanceNamespaceDatabase {
self.namespace
.drop_table(drop_request)
.await
.map_err(|e| map_namespace_lance_error(e, name))?;
.map_err(|e| Error::Runtime {
message: format!("Failed to drop table: {}", e),
})?;
Ok(())
}
@@ -1493,15 +1495,6 @@ mod tests {
.expect("Failed to list tables");
assert!(!table_names_after.contains(&"drop_test".to_string()));
let error = conn
.drop_table("drop_test", &["test_ns".into()])
.await
.expect_err("dropping a missing table should fail");
assert!(
matches!(error, Error::TableNotFound { ref name, .. } if name == "drop_test"),
"expected TableNotFound, got: {error:?}"
);
// Verify: Cannot open dropped table
let open_result = conn.open_table("drop_test").execute().await;
assert!(open_result.is_err());
-3
View File
@@ -15,9 +15,6 @@ use serde_json::Value;
use crate::{Error, Result};
/// Semantic Function type for a Blob v2 value.
pub const FUNCTION_BLOB_V2_TYPE: &str = "blob_v2";
fn invalid_json(error: impl std::fmt::Display) -> Error {
Error::InvalidInput {
message: format!("invalid remote Function JSON: {error}"),
-38
View File
@@ -533,11 +533,6 @@ struct RemoteListJobsResponse {
page_token: Option<String>,
}
#[derive(serde::Deserialize)]
struct RemoteDropFunctionResponse {
dropped: bool,
}
/// Bound on `list_jobs` page walking; a warning is logged when the listing
/// is truncated at this many pages.
const MAX_LIST_JOBS_PAGES: usize = 100;
@@ -588,20 +583,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
response.json().await.err_to_http(request_id)
}
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
let req = self
.client
.post("/v1/functions/drop")
.json(&serde_json::json!({
"name": name,
"version": version,
}));
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?;
Ok(response.dropped)
}
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
self.client.clone(),
@@ -2708,25 +2689,6 @@ mod tests {
assert_eq!(version.version(), "fv_01K3EXACT");
}
#[tokio::test]
async fn test_drop_function_sends_exact_version_and_decodes_replay() {
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/drop");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
body,
serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"})
);
http::Response::builder()
.status(200)
.body(r#"{"dropped":false}"#)
.unwrap()
});
assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap());
}
#[tokio::test]
async fn test_conn_job_waits_to_done() {
let polls = Arc::new(AtomicUsize::new(0));
+33 -345
View File
@@ -29,16 +29,14 @@ use datafusion_common::{ScalarValue, tree_node::TreeNode};
use datafusion_expr::Expr;
use datafusion_physical_plan::PhysicalExpr;
use lance::dataset::NewColumnTransform;
use lance_arrow::{ARROW_EXT_NAME_KEY, BLOB_V2_EXT_NAME, FieldExt};
use lance_core::datatypes::{
BLOB_V2_DESC_FIELD, BlobV2Layout, format_field_path_minimal, parse_field_path,
};
use lance_arrow::FieldExt;
use lance_core::datatypes::{BLOB_V2_DESC_FIELD, format_field_path_minimal, parse_field_path};
use lance_datafusion::planner::Planner;
use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding};
use crate::function::{FunctionApplication, FunctionBinding};
use crate::utils::resolve_arrow_field_path;
use crate::{Error, Result};
@@ -583,23 +581,6 @@ fn resolve_field_path<'a>(schema: &'a ArrowSchema, path: &str) -> Result<Resolve
}
fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
let is_blob_v2 = field
.metadata
.as_ref()
.and_then(|metadata| metadata.get(ARROW_EXT_NAME_KEY))
.map(String::as_str)
== Some(BLOB_V2_EXT_NAME);
if is_blob_v2 {
let arrow_field = lance_namespace::schema::convert_json_arrow_field(field)
.map_err(|e| invalid_function(format!("invalid Function input field: {e}")))?;
if !has_supported_blob_v2_layout(&arrow_field) {
return Err(invalid_function(format!(
"Function input '{}' has an invalid Blob v2 storage layout",
arrow_field.name()
)));
}
return Ok(FUNCTION_BLOB_V2_TYPE.to_string());
}
if field.r#type.fields.is_none() && field.r#type.length.is_none() {
Ok(field.r#type.r#type.clone())
} else {
@@ -609,14 +590,6 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
}
}
fn has_supported_blob_v2_layout(field: &ArrowField) -> bool {
field.is_blob_v2()
&& matches!(
field.data_type(),
DataType::Struct(fields) if BlobV2Layout::classify(fields).is_some()
)
}
/// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
/// any nested `<...>`.
fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
@@ -696,76 +669,6 @@ fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
Ok(data_type)
}
fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonArrowField> {
if raw == FUNCTION_BLOB_V2_TYPE {
return lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
crate::blob(name, nullable),
]))
.map_err(|e| invalid_function(format!("could not encode Blob v2 output field: {e}")))?
.fields
.into_iter()
.next()
.ok_or_else(|| invalid_function("Blob v2 output field is missing"));
}
Ok(JsonArrowField::new(
name.to_string(),
nullable,
parse_output_arrow_type(raw)?,
))
}
fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
expected.name() == actual.name()
&& expected.is_nullable() == actual.is_nullable()
&& if expected.is_blob_v2() {
has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
} else {
function_output_type_matches(expected.data_type(), actual.data_type())
}
}
fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
if expected == actual {
return true;
}
match (expected, actual) {
(DataType::Struct(expected), DataType::Struct(actual)) => {
expected.len() == actual.len()
&& expected
.iter()
.zip(actual)
.all(|(expected, actual)| function_output_field_matches(expected, actual))
}
(DataType::List(expected), DataType::List(actual))
| (DataType::LargeList(expected), DataType::LargeList(actual)) => {
function_output_field_matches(expected, actual)
}
(
DataType::FixedSizeList(expected, expected_size),
DataType::FixedSizeList(actual, actual_size),
) => expected_size == actual_size && function_output_field_matches(expected, actual),
(DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
}
_ => false,
}
}
fn function_output_type_has_blob(data_type: &DataType) -> bool {
match data_type {
DataType::Struct(fields) => fields
.iter()
.any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
DataType::List(field)
| DataType::LargeList(field)
| DataType::FixedSizeList(field, _)
| DataType::Map(field, _) => {
field.is_blob_v2() || function_output_type_has_blob(field.data_type())
}
_ => false,
}
}
fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> {
let mut input_fields = Vec::with_capacity(binding.inputs().len());
for input in binding.inputs() {
@@ -846,18 +749,10 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
binding.binding_id()
)));
}
let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
(has_supported_blob_v2_layout(field), true)
} else {
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
(
function_output_type_matches(&expected_type, field.data_type()),
function_output_type_has_blob(&expected_type),
)
};
if !type_matches {
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
if field.data_type() != &expected_type {
return Err(invalid_function(format!(
"Function output '{}' type no longer matches binding '{}'",
output.output_name,
@@ -886,21 +781,15 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
binding.binding_id()
)));
}
if has_semantic_blob {
output_fields.push(function_output_field(
field.name(),
true,
&output.arrow_type,
)?);
} else {
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
ArrowField::new(field.name().clone(), field.data_type().clone(), true),
]))
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
output_fields.push(json.fields.into_iter().next().unwrap());
}
output_fields.push(ArrowField::new(
field.name().clone(),
field.data_type().clone(),
true,
));
}
let output_schema = JsonArrowSchema::new(output_fields);
let output_schema =
lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(output_fields))
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
let output_schema = serde_json::to_value(output_schema).map_err(|e| {
invalid_function(format!(
"could not encode exact Function output schema: {e}"
@@ -1029,15 +918,16 @@ pub(crate) fn plan_function_application(
"Function logical outputs must be non-nullable during NULL assignment",
));
}
let arrow_type = output.arrow_type.as_deref().ok_or_else(|| {
invalid_function("scalar Function output is missing its Arrow type")
})?;
let data_type =
parse_output_arrow_type(output.arrow_type.as_deref().ok_or_else(|| {
invalid_function("scalar Function output is missing its Arrow type")
})?)?;
outputs.push(FunctionOutputTarget {
result_field: WHOLE_RESULT_FIELD.to_string(),
output_name: name.to_string(),
output_ordinal: 0,
});
output_fields.push(function_output_field(name, true, arrow_type)?);
output_fields.push(JsonArrowField::new(name.to_string(), true, data_type));
}
"named_struct" => {
if output.fields.is_empty() {
@@ -1081,7 +971,13 @@ pub(crate) fn plan_function_application(
let fields = output
.fields
.iter()
.map(|field| function_output_field(&field.name, false, &field.arrow_type))
.map(|field| {
Ok(JsonArrowField::new(
field.name.clone(),
false,
parse_output_arrow_type(&field.arrow_type)?,
))
})
.collect::<Result<Vec<_>>>()?;
let mut data_type = JsonArrowDataType::new("struct".to_string());
data_type.fields = Some(fields);
@@ -1108,7 +1004,11 @@ pub(crate) fn plan_function_application(
output_name: name.clone(),
output_ordinal: ordinal as u32,
});
output_fields.push(function_output_field(name, true, &field.arrow_type)?);
output_fields.push(JsonArrowField::new(
name.clone(),
true,
parse_output_arrow_type(&field.arrow_type)?,
));
}
}
}
@@ -1745,7 +1645,7 @@ mod tests {
}
use arrow_array::record_batch;
use arrow_schema::{DataType, TimeUnit};
use arrow_schema::DataType;
use futures::TryStreamExt;
use lance::dataset::ColumnAlteration;
@@ -2706,73 +2606,6 @@ mod tests {
.unwrap()
}
fn blob_application(output: &str) -> FunctionApplication {
FunctionApplication::from_json(&format!(
r#"{{
"function":{{"name":"blob_features","version":"fv_blob"}},
"inputs":[
{{"parameter":"image","kind":"column","value":{{"path":"image"}}}}
],
"output":{output}
}}"#
))
.unwrap()
}
fn binding_from_plan(plan: &FunctionDeclarationPlan) -> FunctionBinding {
let inputs = plan
.input_bindings
.iter()
.enumerate()
.map(|(index, input)| {
serde_json::json!({
"parameter": input.parameter,
"field_id": index,
"field_path": input.field_path,
"arrow_type": input.arrow_type,
"nullable": input.nullable,
})
})
.collect::<Vec<_>>();
let outputs = plan
.outputs
.iter()
.zip(&plan.output_schema.fields)
.enumerate()
.map(|(index, (output, field))| {
serde_json::json!({
"result_field": output.result_field,
"output_name": output.output_name,
"output_field_id": 100 + index,
"output_ordinal": output.output_ordinal,
"arrow_type": canonical_input_arrow_type(field).unwrap(),
"nullable": false,
})
})
.collect::<Vec<_>>();
FunctionBinding::from_json(
&serde_json::json!({
"binding_id": "fb_blob",
"function": plan.application.function(),
"inputs": inputs,
"outputs": outputs,
"input_schema": plan.input_schema,
"output_schema": plan.output_schema,
})
.to_string(),
)
.unwrap()
}
fn full_blob_field(name: &str, nullable: bool) -> ArrowField {
ArrowField::new(
name,
DataType::Struct(lance_core::datatypes::BLOB_V2_LOGICAL_FIELDS.clone()),
nullable,
)
.with_metadata(crate::blob(name, nullable).metadata().clone())
}
fn function_binding_schema(title_nullable: bool, body_nullable: bool) -> ArrowSchema {
ArrowSchema::new(vec![
ArrowField::new("title", DataType::Utf8, title_nullable),
@@ -3046,151 +2879,6 @@ mod tests {
);
}
#[test]
fn test_blob_function_plans_semantic_input_and_scalar_output() {
let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
let application =
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
let plan = plan_function_application(&schema, &application, Some("thumbnail")).unwrap();
assert_eq!(plan.input_bindings[0].arrow_type, FUNCTION_BLOB_V2_TYPE);
let input_schema =
lance_namespace::schema::convert_json_arrow_schema(&plan.input_schema).unwrap();
assert!(input_schema.field(0).is_blob_v2());
let output_schema =
lance_namespace::schema::convert_json_arrow_schema(&plan.output_schema).unwrap();
assert!(output_schema.field(0).is_blob_v2());
}
#[test]
fn test_blob_scalar_binding_accepts_full_logical_layout() {
let input = crate::blob("image", false);
let application =
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
let plan = plan_function_application(
&ArrowSchema::new(vec![input.clone()]),
&application,
Some("thumbnail"),
)
.unwrap();
let binding = binding_from_plan(&plan);
let mut metadata = full_blob_field("thumbnail", true).metadata().clone();
metadata.extend(function_computed_column_metadata(
binding.binding_id(),
0,
&["image".into()],
));
let output = full_blob_field("thumbnail", true).with_metadata(metadata);
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, output]), &binding).unwrap();
}
#[test]
fn test_blob_binding_rejects_marker_on_invalid_storage_layout() {
let input = crate::blob("image", false);
let application =
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
let plan = plan_function_application(
&ArrowSchema::new(vec![input.clone()]),
&application,
Some("thumbnail"),
)
.unwrap();
let binding = binding_from_plan(&plan);
let malformed = ArrowField::new("thumbnail", DataType::Int64, true)
.with_metadata(crate::blob("thumbnail", true).metadata().clone());
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, malformed]), &binding)
.unwrap_err();
}
#[test]
fn test_blob_input_rejects_marker_on_invalid_storage_layout() {
let malformed = ArrowField::new("image", DataType::Int64, false)
.with_metadata(crate::blob("image", false).metadata().clone());
let application =
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
plan_function_application(
&ArrowSchema::new(vec![malformed]),
&application,
Some("thumbnail"),
)
.unwrap_err();
}
#[test]
fn test_non_blob_input_does_not_require_json_round_trip() {
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
ArrowField::new("event_time", DataType::Time64(TimeUnit::Microsecond), false),
]))
.unwrap();
assert_eq!(
canonical_input_arrow_type(&json.fields[0]).unwrap(),
"time64"
);
}
#[test]
fn test_blob_named_struct_plans_expanded_and_whole_outputs() {
let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
let application = blob_application(
r#"{"kind":"named_struct","fields":[
{"name":"thumbnail","arrow_type":"blob_v2","nullable":false},
{"name":"width","arrow_type":"int32","nullable":false}
]}"#,
);
let expanded = plan_function_application(&schema, &application, None).unwrap();
let expanded_schema =
lance_namespace::schema::convert_json_arrow_schema(&expanded.output_schema).unwrap();
assert!(expanded_schema.field(0).is_blob_v2());
assert_eq!(expanded_schema.field(1).data_type(), &DataType::Int32);
let whole = plan_function_application(&schema, &application, Some("analysis")).unwrap();
let whole_schema =
lance_namespace::schema::convert_json_arrow_schema(&whole.output_schema).unwrap();
let DataType::Struct(fields) = whole_schema.field(0).data_type() else {
panic!("whole Function output should be a struct");
};
assert!(fields[0].is_blob_v2());
assert_eq!(fields[1].data_type(), &DataType::Int32);
}
#[test]
fn test_blob_whole_struct_binding_accepts_full_logical_layout() {
let input = crate::blob("image", false);
let application = blob_application(
r#"{"kind":"named_struct","fields":[
{"name":"thumbnail","arrow_type":"blob_v2","nullable":false},
{"name":"width","arrow_type":"int32","nullable":false}
]}"#,
);
let plan = plan_function_application(
&ArrowSchema::new(vec![input.clone()]),
&application,
Some("analysis"),
)
.unwrap();
let binding = binding_from_plan(&plan);
let output = ArrowField::new(
"analysis",
DataType::Struct(Fields::from(vec![
full_blob_field("thumbnail", false),
ArrowField::new("width", DataType::Int32, false),
])),
true,
)
.with_metadata(function_computed_column_metadata(
binding.binding_id(),
0,
&["image".into()],
));
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, output]), &binding).unwrap();
}
#[test]
fn test_function_mapping_and_sibling_collisions_fail_before_request() {
let unknown = named_struct_application(r#"{"missing":"renamed"}"#);
@@ -45,11 +45,7 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
.get_function("normalize_score", "fv_exact")
.await
.unwrap_err();
let drop_error = connection
.drop_function("normalize_score", "fv_exact")
.await
.unwrap_err();
for error in [create_error, lookup_error, drop_error] {
for error in [create_error, lookup_error] {
assert!(matches!(
error,
Error::NotSupported { message }
@@ -78,12 +78,6 @@
"type": "utf8"
}
},
{
"arrow_type": "large_utf8",
"json": {
"type": "large_utf8"
}
},
{
"arrow_type": "binary",
"json": {
@@ -177,21 +171,6 @@
]
}
},
{
"arrow_type": "list<large_utf8>",
"json": {
"type": "list",
"fields": [
{
"name": "item",
"nullable": false,
"type": {
"type": "large_utf8"
}
}
]
}
},
{
"arrow_type": "large_list<utf8>",
"json": {
@@ -207,21 +186,6 @@
]
}
},
{
"arrow_type": "large_list<large_utf8>",
"json": {
"type": "large_list",
"fields": [
{
"name": "item",
"nullable": false,
"type": {
"type": "large_utf8"
}
}
]
}
},
{
"arrow_type": "fixed_size_list<float32, 384>",
"json": {
@@ -366,4 +330,4 @@
"timestamp[us]",
"struct<a: int32>"
]
}
}