mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-01 11:08:55 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cad46fc683 |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.38.0"
|
||||
current_version = "0.38.0-beta.14"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
Generated
+27
-29
@@ -535,9 +535,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-trait"
|
||||
version = "0.1.92"
|
||||
version = "0.1.91"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
||||
checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1443,9 +1443,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.25.2"
|
||||
version = "1.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
|
||||
checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec"
|
||||
dependencies = [
|
||||
"bytemuck_derive",
|
||||
]
|
||||
@@ -5402,7 +5402,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb"
|
||||
version = "0.38.0"
|
||||
version = "0.38.0-beta.13"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"anyhow",
|
||||
@@ -5490,7 +5490,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-nodejs"
|
||||
version = "0.38.0"
|
||||
version = "0.38.0-beta.13"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5515,7 +5515,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0"
|
||||
version = "0.38.0-beta.13"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5748,9 +5748,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
version = "0.4.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "loom"
|
||||
@@ -6001,9 +6001,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "moka"
|
||||
version = "0.12.16"
|
||||
version = "0.12.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9"
|
||||
checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046"
|
||||
dependencies = [
|
||||
"async-lock",
|
||||
"crossbeam-channel",
|
||||
@@ -6097,15 +6097,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi"
|
||||
version = "3.12.2"
|
||||
version = "3.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58c5f4d5375213fdb7be2655e152386e82f026f9a5ba36a75556e11359aafe09"
|
||||
checksum = "de33522036981030a75c231829566bc63414e08101a6f5ff4ac6cef19c8e0941"
|
||||
dependencies = [
|
||||
"bitflags 2.11.1",
|
||||
"chrono",
|
||||
"ctor 1.0.12",
|
||||
"futures",
|
||||
"libc",
|
||||
"napi-build",
|
||||
"napi-sys",
|
||||
"nohash-hasher",
|
||||
@@ -6117,15 +6116,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi-build"
|
||||
version = "2.4.1"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab"
|
||||
checksum = "5282704fbe8d49b0cf8b08e3f33233416a528658f205c7e5ace63b582de0b11c"
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive"
|
||||
version = "3.6.3"
|
||||
version = "3.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af"
|
||||
checksum = "4d5c9c02556ea6dc99dffd36c1ce60141411657438501a125b675776d011ce92"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"ctor 1.0.12",
|
||||
@@ -6137,9 +6136,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "napi-derive-backend"
|
||||
version = "6.1.2"
|
||||
version = "6.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae"
|
||||
checksum = "d60b5d773ad46c698c8cc2cd9fde0b283d39cbb7f71c04bee633c7bdba4423bd"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"proc-macro2",
|
||||
@@ -8602,9 +8601,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "roaring"
|
||||
version = "0.11.5"
|
||||
version = "0.11.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1"
|
||||
checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
@@ -9064,9 +9063,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with"
|
||||
version = "3.22.0"
|
||||
version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a"
|
||||
checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bs58",
|
||||
@@ -9074,7 +9073,6 @@ dependencies = [
|
||||
"hex",
|
||||
"indexmap 1.9.3",
|
||||
"indexmap 2.14.0",
|
||||
"jiff",
|
||||
"schemars 0.9.0",
|
||||
"schemars 1.2.1",
|
||||
"serde_core",
|
||||
@@ -9085,9 +9083,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_with_macros"
|
||||
version = "3.22.0"
|
||||
version = "3.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46"
|
||||
checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
@@ -10454,9 +10452,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.26.0"
|
||||
version = "1.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812"
|
||||
checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"js-sys",
|
||||
|
||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
||||
<dependency>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-core</artifactId>
|
||||
<version>0.38.0</version>
|
||||
<version>0.38.0-beta.14</version>
|
||||
</dependency>
|
||||
```
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<parent>
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-final.0</version>
|
||||
<version>0.38.0-beta.14</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
|
||||
<groupId>com.lancedb</groupId>
|
||||
<artifactId>lancedb-parent</artifactId>
|
||||
<version>0.38.0-final.0</version>
|
||||
<version>0.38.0-beta.14</version>
|
||||
<packaging>pom</packaging>
|
||||
<name>${project.artifactId}</name>
|
||||
<description>LanceDB Java SDK Parent POM</description>
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
[package]
|
||||
name = "lancedb-nodejs"
|
||||
edition.workspace = true
|
||||
version = "0.38.0"
|
||||
version = "0.38.0-beta.14"
|
||||
publish = false
|
||||
license.workspace = true
|
||||
description.workspace = true
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-darwin-arm64",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": ["darwin"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.darwin-arm64.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": ["linux"],
|
||||
"cpu": ["arm64"],
|
||||
"main": "lancedb.linux-arm64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-gnu.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": ["linux"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.linux-x64-musl.node",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"os": ["win32"],
|
||||
"cpu": ["x64"],
|
||||
"main": "lancedb.win32-x64-msvc.node",
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@
|
||||
"ann"
|
||||
],
|
||||
"private": false,
|
||||
"version": "0.38.0",
|
||||
"version": "0.38.0-beta.14",
|
||||
"main": "dist/index.js",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
|
||||
+1
-5
@@ -664,11 +664,7 @@ impl JsFullTextQuery {
|
||||
}
|
||||
|
||||
fn parse_fts_query(query: Object) -> napi::Result<FullTextSearchQuery> {
|
||||
// `&JsFullTextQuery` recovers a native class reference through napi's borrow-tracked
|
||||
// path, which is only usable from generated `#[napi]` argument conversion. This is a
|
||||
// manual lookup on a nested `Object` property instead, so use `ClassInstance`, which
|
||||
// unwraps the class without requiring a borrow scope.
|
||||
if let Ok(Some(query)) = query.get::<ClassInstance<JsFullTextQuery>>("query") {
|
||||
if let Ok(Some(query)) = query.get::<&JsFullTextQuery>("query") {
|
||||
Ok(FullTextSearchQuery::new_query(query.inner.clone()))
|
||||
} else if let Ok(Some(query_text)) = query.get::<String>("query") {
|
||||
let mut query_text = query_text;
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.38.0"
|
||||
version = "0.38.0-beta.14"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
|
||||
@@ -150,7 +150,6 @@ class Connection(object):
|
||||
def job(self, job_id: str) -> Job: ...
|
||||
async def create_function_async(self, request_json: str) -> Job: ...
|
||||
async def get_function(self, name: str, version: str) -> str: ...
|
||||
async def drop_function(self, name: str, version: str) -> bool: ...
|
||||
async def list_jobs(self) -> List[JobInfo]: ...
|
||||
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
|
||||
async def cancel_job(self, job_id: str) -> bool: ...
|
||||
|
||||
@@ -712,16 +712,6 @@ class DBConnection(EnforceOverrides):
|
||||
"Function catalog operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def drop_function(self, name: str, *, version: str) -> bool:
|
||||
"""Drop one exact immutable Function version from the remote catalog.
|
||||
|
||||
Returns True when the version changed to Dropped and False for an
|
||||
idempotent replay. Local connections raise NotImplementedError.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Function catalog operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def job(self, job_id: str) -> Job:
|
||||
"""A [Job][lancedb.job.Job] handle for a server-side job by id.
|
||||
|
||||
@@ -1423,10 +1413,6 @@ class LanceDBConnection(DBConnection):
|
||||
def get_function(self, name: str, *, version: str) -> FunctionVersion:
|
||||
return LOOP.run(self._conn.get_function(name, version=version))
|
||||
|
||||
@override
|
||||
def drop_function(self, name: str, *, version: str) -> bool:
|
||||
return LOOP.run(self._conn.drop_function(name, version=version))
|
||||
|
||||
@override
|
||||
def list_jobs(self) -> List[JobInfo]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
@@ -2257,10 +2243,6 @@ class AsyncConnection(object):
|
||||
"""Open one exact immutable Function version from the remote catalog."""
|
||||
return FunctionVersion.from_json(await self._inner.get_function(name, version))
|
||||
|
||||
async def drop_function(self, name: str, *, version: str) -> bool:
|
||||
"""Drop one exact immutable Function version from the remote catalog."""
|
||||
return await self._inner.drop_function(name, version)
|
||||
|
||||
async def list_jobs(self) -> List[JobInfo]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return await self._inner.list_jobs()
|
||||
|
||||
@@ -49,8 +49,6 @@ from pydantic import (
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from .schema import is_blob_v2_field as _is_blob_v2_field
|
||||
|
||||
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
|
||||
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
|
||||
_UInt64 = conint(strict=True, ge=0, le=2**64 - 1)
|
||||
@@ -520,7 +518,6 @@ class RefreshColumnResult(_RemoteValue):
|
||||
|
||||
|
||||
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
|
||||
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
|
||||
|
||||
|
||||
_GRAMMAR_PRIMITIVES = (
|
||||
@@ -537,7 +534,6 @@ _GRAMMAR_PRIMITIVES = (
|
||||
(pa.float32(), "float32"),
|
||||
(pa.float64(), "float64"),
|
||||
(pa.string(), "utf8"),
|
||||
(pa.large_string(), "large_utf8"),
|
||||
(pa.binary(), "binary"),
|
||||
(pa.date32(), "date32"),
|
||||
(pa.date64(), "date64"),
|
||||
@@ -585,90 +581,20 @@ def _validate_exact_arrow_field(field: pa.Field) -> None:
|
||||
"unsupported Arrow type for Function signature: field names "
|
||||
"must not be empty"
|
||||
)
|
||||
if _is_blob_v2_field(field):
|
||||
if not _has_supported_blob_v2_layout(field):
|
||||
raise TypeError(
|
||||
"unsupported Arrow type for Function signature: lance.blob.v2 "
|
||||
f"requires a supported Blob storage layout, got {field}"
|
||||
)
|
||||
elif field.metadata:
|
||||
if field.metadata:
|
||||
raise TypeError(
|
||||
"unsupported Arrow type for Function signature: field metadata "
|
||||
f"is not supported, got {field}"
|
||||
)
|
||||
|
||||
|
||||
def _has_supported_blob_v2_layout(field: pa.Field) -> bool:
|
||||
data_type = field.type
|
||||
if isinstance(data_type, pa.ExtensionType):
|
||||
data_type = data_type.storage_type
|
||||
if not pa.types.is_struct(data_type):
|
||||
return False
|
||||
|
||||
fields = tuple(data_type)
|
||||
|
||||
def matches(spec, compare_nullable) -> bool:
|
||||
return len(fields) == len(spec) and all(
|
||||
actual.name == name
|
||||
and actual.type == expected_type
|
||||
and (not check_nullable or actual.nullable == nullable)
|
||||
for actual, (name, expected_type, nullable), check_nullable in zip(
|
||||
fields, spec, compare_nullable
|
||||
)
|
||||
)
|
||||
|
||||
logical_minimal = (
|
||||
("data", pa.large_binary(), True),
|
||||
("uri", pa.utf8(), True),
|
||||
)
|
||||
logical_full = logical_minimal + (
|
||||
("position", pa.uint64(), True),
|
||||
("size", pa.uint64(), True),
|
||||
)
|
||||
prepared = (
|
||||
("kind", pa.uint8(), True),
|
||||
("data", pa.large_binary(), True),
|
||||
("uri", pa.utf8(), True),
|
||||
("blob_id", pa.uint32(), True),
|
||||
("blob_size", pa.uint64(), True),
|
||||
("position", pa.uint64(), True),
|
||||
)
|
||||
descriptor = (
|
||||
("kind", pa.uint8(), False),
|
||||
("position", pa.uint64(), False),
|
||||
("size", pa.uint64(), False),
|
||||
("blob_id", pa.uint32(), False),
|
||||
("blob_uri", pa.utf8(), False),
|
||||
)
|
||||
return (
|
||||
matches(logical_minimal, (True, True))
|
||||
or matches(logical_full, (True, True, False, False))
|
||||
or matches(prepared, (True,) * len(prepared))
|
||||
or matches(descriptor, (False,) * len(descriptor))
|
||||
)
|
||||
|
||||
|
||||
def _canonical_arrow_field(field: pa.Field) -> str:
|
||||
_validate_exact_arrow_field(field)
|
||||
if _is_blob_v2_field(field):
|
||||
return _FUNCTION_BLOB_V2_TYPE
|
||||
return _canonical_arrow_type(field.type)
|
||||
|
||||
|
||||
def _exact_arrow_field(field: pa.Field) -> dict[str, Any]:
|
||||
_validate_exact_arrow_field(field)
|
||||
if _is_blob_v2_field(field):
|
||||
raise TypeError(
|
||||
"unsupported Arrow type for Function signature: nested Blob v2 "
|
||||
"fields are not supported; declare Blob parameters or named result "
|
||||
"fields directly"
|
||||
)
|
||||
value = {
|
||||
return {
|
||||
"name": field.name,
|
||||
"nullable": field.nullable,
|
||||
"type": _exact_arrow_type(field.type),
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def _exact_arrow_type(data_type: pa.DataType) -> dict[str, Any]:
|
||||
@@ -792,11 +718,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
|
||||
if output.metadata:
|
||||
raise TypeError("Function output schema metadata is not supported")
|
||||
fields = tuple(output)
|
||||
elif (
|
||||
isinstance(output, pa.Field)
|
||||
and not _is_blob_v2_field(output)
|
||||
and pa.types.is_struct(output.type)
|
||||
):
|
||||
elif isinstance(output, pa.Field) and pa.types.is_struct(output.type):
|
||||
_validate_exact_arrow_field(output)
|
||||
if output.nullable:
|
||||
raise ValueError("Function output must be non-nullable")
|
||||
@@ -818,7 +740,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
|
||||
raise ValueError("Function output must be non-nullable")
|
||||
return FunctionOutput(
|
||||
kind="scalar",
|
||||
arrow_type=_canonical_arrow_field(field),
|
||||
arrow_type=_canonical_arrow_type(field.type),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
@@ -836,7 +758,7 @@ def _function_output(output: pa.DataType | pa.Field | pa.Schema) -> FunctionOutp
|
||||
fields=tuple(
|
||||
FunctionResultField(
|
||||
name=field.name,
|
||||
arrow_type=_canonical_arrow_field(field),
|
||||
arrow_type=_canonical_arrow_type(field.type),
|
||||
nullable=False,
|
||||
)
|
||||
for field in fields
|
||||
@@ -870,7 +792,7 @@ def _infer_signature(
|
||||
inputs = tuple(
|
||||
FunctionParameter(
|
||||
name=field.name,
|
||||
arrow_type=_canonical_arrow_field(field),
|
||||
arrow_type=_canonical_arrow_type(field.type),
|
||||
nullable=field.nullable,
|
||||
)
|
||||
for field in input_schema
|
||||
@@ -893,9 +815,7 @@ def _infer_signature(
|
||||
inputs.append(
|
||||
FunctionParameter(
|
||||
name=parameter.name,
|
||||
arrow_type=_canonical_arrow_field(
|
||||
pa.field(parameter.name, data_type, nullable=nullable)
|
||||
),
|
||||
arrow_type=_canonical_arrow_type(data_type),
|
||||
nullable=nullable,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -749,10 +749,6 @@ class RemoteDBConnection(DBConnection):
|
||||
def get_function(self, name: str, *, version: str) -> FunctionVersion:
|
||||
return LOOP.run(self._conn.get_function(name, version=version))
|
||||
|
||||
@override
|
||||
def drop_function(self, name: str, *, version: str) -> bool:
|
||||
return LOOP.run(self._conn.drop_function(name, version=version))
|
||||
|
||||
@override
|
||||
def list_jobs(self) -> List["JobInfo"]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
|
||||
@@ -12,8 +12,6 @@ from datetime import date
|
||||
import http.server
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
@@ -21,13 +19,7 @@ import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb.functions import (
|
||||
PythonRuntimeSpec,
|
||||
UdfDefinition,
|
||||
_canonical_arrow_type,
|
||||
_GRAMMAR_PRIMITIVES,
|
||||
udf,
|
||||
)
|
||||
from lancedb.functions import PythonRuntimeSpec, UdfDefinition, udf
|
||||
|
||||
THRESHOLD = 20
|
||||
_CACHE = None
|
||||
@@ -69,80 +61,6 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
|
||||
}
|
||||
|
||||
|
||||
def _main_udf_source(
|
||||
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
|
||||
) -> str:
|
||||
return (
|
||||
"from __future__ import annotations\n"
|
||||
"from lancedb.functions import udf\n"
|
||||
f"THRESHOLD = {threshold}\n"
|
||||
"\n"
|
||||
"@udf\n"
|
||||
f"def label(value: {input_annotation}) -> str:\n"
|
||||
f" return 'big' if value {comparison} THRESHOLD else 'small'\n"
|
||||
"\n"
|
||||
"assert label.__module__ == '__main__'\n"
|
||||
"print(label.registration_request.to_canonical_json())\n"
|
||||
)
|
||||
|
||||
|
||||
def _run_main_udf(path: Path, source: str) -> dict:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(source)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(path)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def test_main_udf_registration_identity_is_stable_across_processes_and_paths(
|
||||
tmp_path,
|
||||
):
|
||||
source = _main_udf_source()
|
||||
original_path = tmp_path / "original" / "job.py"
|
||||
moved_path = tmp_path / "moved" / "renamed_job.py"
|
||||
|
||||
original_runs = [_run_main_udf(original_path, source) for _ in range(2)]
|
||||
moved_run = _run_main_udf(moved_path, source)
|
||||
|
||||
assert len({run["artifact"]["digest"] for run in [*original_runs, moved_run]}) == 1
|
||||
assert all(
|
||||
run["signature"] == original_runs[0]["signature"]
|
||||
for run in [original_runs[1], moved_run]
|
||||
)
|
||||
assert original_runs[0] == original_runs[1] == moved_run
|
||||
|
||||
body_change = _run_main_udf(
|
||||
tmp_path / "changes" / "body.py", _main_udf_source(comparison=">")
|
||||
)
|
||||
global_change = _run_main_udf(
|
||||
tmp_path / "changes" / "global.py", _main_udf_source(threshold=21)
|
||||
)
|
||||
annotation_change = _run_main_udf(
|
||||
tmp_path / "changes" / "annotation.py",
|
||||
_main_udf_source(input_annotation="float"),
|
||||
)
|
||||
|
||||
baseline = original_runs[0]
|
||||
assert baseline["signature"] == body_change["signature"]
|
||||
assert baseline["signature"] == global_change["signature"]
|
||||
assert baseline["signature"] != annotation_change["signature"]
|
||||
assert (
|
||||
len(
|
||||
{
|
||||
baseline["artifact"]["digest"],
|
||||
body_change["artifact"]["digest"],
|
||||
global_change["artifact"]["digest"],
|
||||
annotation_change["artifact"]["digest"],
|
||||
}
|
||||
)
|
||||
== 4
|
||||
)
|
||||
|
||||
|
||||
def _run_packaged(definition, *args):
|
||||
"""Execute the shipped artifact in a fresh namespace, as a worker would."""
|
||||
source = base64.b64decode(definition.registration_request.artifact.content.data)
|
||||
@@ -303,6 +221,8 @@ def test_udf_resolves_module_globals_before_builtins(tmp_path):
|
||||
|
||||
|
||||
def test_canonical_arrow_type_prefers_the_compact_grammar():
|
||||
from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type
|
||||
|
||||
golden = json.loads(
|
||||
(
|
||||
Path(__file__).parents[3]
|
||||
@@ -323,6 +243,7 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
|
||||
for outside in [
|
||||
pa.timestamp("us"),
|
||||
pa.decimal128(10, 2),
|
||||
pa.large_string(),
|
||||
pa.large_binary(),
|
||||
pa.binary(4),
|
||||
pa.duration("s"),
|
||||
@@ -516,6 +437,8 @@ def test_udf_recursion_versus_a_rebound_module_name(tmp_path):
|
||||
|
||||
|
||||
def test_canonical_arrow_type_uses_exact_json_for_list_child_properties():
|
||||
from lancedb.functions import _canonical_arrow_type
|
||||
|
||||
nullable = pa.list_(pa.float32())
|
||||
assert json.loads(_canonical_arrow_type(nullable)) == {
|
||||
"type": "list",
|
||||
@@ -605,7 +528,6 @@ def _arrow_type_from_golden(spec: dict) -> pa.DataType:
|
||||
"null": pa.null(),
|
||||
"bool": pa.bool_(),
|
||||
"utf8": pa.string(),
|
||||
"large_utf8": pa.large_string(),
|
||||
"binary": pa.binary(),
|
||||
"float16": pa.float16(),
|
||||
"float32": pa.float32(),
|
||||
@@ -622,6 +544,8 @@ def test_arrow_type_grammar_matches_the_shared_golden():
|
||||
/ "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json"
|
||||
).read_text()
|
||||
)
|
||||
from lancedb.functions import _canonical_arrow_type
|
||||
|
||||
emitted = {
|
||||
case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"]))
|
||||
for case in golden["valid"]
|
||||
@@ -654,151 +578,6 @@ def test_explicit_arrow_schema_is_deterministic():
|
||||
assert signature.output.nullable is False
|
||||
|
||||
|
||||
def test_blob_fields_use_the_scalar_function_semantic_type():
|
||||
@udf(
|
||||
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
|
||||
output_schema=lancedb.blob("result", nullable=False),
|
||||
)
|
||||
def copy_blob(image):
|
||||
return image
|
||||
|
||||
signature = copy_blob.registration_request.signature
|
||||
assert signature.inputs[0].arrow_type == "blob_v2"
|
||||
assert signature.output.kind == "scalar"
|
||||
assert signature.output.arrow_type == "blob_v2"
|
||||
|
||||
|
||||
def test_named_struct_function_can_include_a_blob_result_field():
|
||||
@udf(
|
||||
input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
|
||||
output_schema=pa.schema(
|
||||
[
|
||||
lancedb.blob("thumbnail", nullable=False),
|
||||
pa.field("width", pa.int32(), nullable=False),
|
||||
]
|
||||
),
|
||||
)
|
||||
def inspect_blob(image):
|
||||
return {"thumbnail": image, "width": 1}
|
||||
|
||||
output = inspect_blob.registration_request.signature.output
|
||||
assert output.kind == "named_struct"
|
||||
assert [(field.name, field.arrow_type) for field in output.fields] == [
|
||||
("thumbnail", "blob_v2"),
|
||||
("width", "int32"),
|
||||
]
|
||||
|
||||
|
||||
def test_metadata_marked_blob_field_uses_the_semantic_type():
|
||||
extension = lancedb.blob("image", nullable=False).type
|
||||
storage = (
|
||||
extension.storage_type if isinstance(extension, pa.ExtensionType) else extension
|
||||
)
|
||||
metadata_blob = pa.field(
|
||||
"image",
|
||||
storage,
|
||||
nullable=False,
|
||||
metadata={"ARROW:extension:name": "lance.blob.v2"},
|
||||
)
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([metadata_blob]),
|
||||
output_schema=pa.field("size", pa.int64(), nullable=False),
|
||||
)
|
||||
def blob_size(image):
|
||||
return len(image)
|
||||
|
||||
assert blob_size.registration_request.signature.inputs[0].arrow_type == "blob_v2"
|
||||
|
||||
|
||||
def test_blob_marker_rejects_invalid_storage_layout():
|
||||
malformed = pa.field(
|
||||
"image",
|
||||
pa.int64(),
|
||||
nullable=False,
|
||||
metadata={"ARROW:extension:name": "lance.blob.v2"},
|
||||
)
|
||||
|
||||
with pytest.raises(TypeError, match="requires a supported Blob storage layout"):
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([malformed]),
|
||||
output_schema=pa.field("size", pa.int64(), nullable=False),
|
||||
)
|
||||
def blob_size(image):
|
||||
return len(image)
|
||||
|
||||
|
||||
def test_nested_blob_signature_field_has_a_clear_error():
|
||||
nested = pa.field(
|
||||
"value",
|
||||
pa.struct([lancedb.blob("image", nullable=False)]),
|
||||
nullable=False,
|
||||
)
|
||||
with pytest.raises(TypeError, match="nested Blob v2 fields are not supported"):
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([nested]),
|
||||
output_schema=pa.field("size", pa.int64(), nullable=False),
|
||||
)
|
||||
def blob_size(value):
|
||||
return len(value["image"])
|
||||
|
||||
|
||||
def test_nested_non_blob_extension_is_not_silently_unwrapped():
|
||||
class TestExtension(pa.ExtensionType):
|
||||
def __init__(self):
|
||||
super().__init__(pa.int64(), "test.function.extension")
|
||||
|
||||
def __arrow_ext_serialize__(self):
|
||||
return b""
|
||||
|
||||
@classmethod
|
||||
def __arrow_ext_deserialize__(cls, storage_type, serialized):
|
||||
return cls()
|
||||
|
||||
nested = pa.field(
|
||||
"value",
|
||||
pa.struct([pa.field("extended", TestExtension(), nullable=False)]),
|
||||
nullable=False,
|
||||
)
|
||||
with pytest.raises(TypeError, match="unsupported Arrow type"):
|
||||
|
||||
@udf(
|
||||
input_schema=pa.schema([nested]),
|
||||
output_schema=pa.field("result", pa.int64(), nullable=False),
|
||||
)
|
||||
def extension_value(value):
|
||||
return value["extended"]
|
||||
|
||||
|
||||
def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
|
||||
input_schema = pa.schema([pa.field("text", pa.large_string(), nullable=True)])
|
||||
output_schema = pa.field("result", pa.large_string(), nullable=False)
|
||||
|
||||
@udf(input_schema=input_schema, output_schema=output_schema)
|
||||
def preserve(text):
|
||||
return text
|
||||
|
||||
signature = preserve.registration_request.signature
|
||||
assert signature.inputs[0].arrow_type == "large_utf8"
|
||||
assert signature.inputs[0].nullable is True
|
||||
assert signature.output.arrow_type == "large_utf8"
|
||||
assert signature.output.nullable is False
|
||||
|
||||
nested = pa.struct([pa.field("text", pa.large_string(), nullable=True)])
|
||||
assert json.loads(_canonical_arrow_type(nested)) == {
|
||||
"type": "struct",
|
||||
"fields": [
|
||||
{
|
||||
"name": "text",
|
||||
"nullable": True,
|
||||
"type": {"type": "large_utf8"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_nested_struct_output_uses_canonical_exact_json():
|
||||
token = pa.struct(
|
||||
[
|
||||
@@ -1017,8 +796,6 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path):
|
||||
db.create_function_async(normalize_score)
|
||||
with pytest.raises(NotImplementedError, match=message):
|
||||
db.get_function("normalize_score", version="fv_exact")
|
||||
with pytest.raises(NotImplementedError, match=message):
|
||||
db.drop_function("normalize_score", version="fv_exact")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
@@ -1064,12 +841,6 @@ def _mock_remote_function_catalog():
|
||||
"version": "fv_exact",
|
||||
}
|
||||
response = state["version"]
|
||||
elif self.path == "/v1/functions/drop":
|
||||
assert body == {
|
||||
"name": "normalize_score",
|
||||
"version": "fv_exact",
|
||||
}
|
||||
response = {"dropped": True}
|
||||
else:
|
||||
status = 404
|
||||
response = {"error": "not found"}
|
||||
@@ -1128,40 +899,3 @@ def test_blocking_remote_registration_returns_function_version():
|
||||
"/v1/functions/create",
|
||||
"/v1/jobs/describe",
|
||||
]
|
||||
|
||||
|
||||
def test_remote_drop_function_sends_exact_version():
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = lancedb.connect(
|
||||
"db://dev",
|
||||
api_key="fake",
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
assert db.drop_function("normalize_score", version="fv_exact") is True
|
||||
|
||||
assert state["requests"] == [
|
||||
(
|
||||
"/v1/functions/drop",
|
||||
{"name": "normalize_score", "version": "fv_exact"},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_remote_drop_function_sends_exact_version():
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = await lancedb.connect_async(
|
||||
"db://dev",
|
||||
api_key="fake",
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
assert await db.drop_function("normalize_score", version="fv_exact") is True
|
||||
|
||||
assert state["requests"] == [
|
||||
(
|
||||
"/v1/functions/drop",
|
||||
{"name": "normalize_score", "version": "fv_exact"},
|
||||
)
|
||||
]
|
||||
|
||||
@@ -629,17 +629,6 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn drop_function(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
version: String,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner.drop_function(name, version).await.infer_error()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
future_into_py(self_.py(), async move {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb"
|
||||
version = "0.38.0"
|
||||
version = "0.38.0-beta.14"
|
||||
edition.workspace = true
|
||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||
license.workspace = true
|
||||
|
||||
@@ -523,21 +523,6 @@ impl Connection {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Drop one exact immutable Function version from the remote catalog.
|
||||
///
|
||||
/// Returns `true` when the server appended a Dropped transition and
|
||||
/// `false` for an idempotent replay. Local databases return
|
||||
/// [`Error::NotSupported`].
|
||||
pub async fn drop_function(
|
||||
&self,
|
||||
name: impl AsRef<str>,
|
||||
version: impl AsRef<str>,
|
||||
) -> Result<bool> {
|
||||
self.internal
|
||||
.drop_function(name.as_ref(), version.as_ref())
|
||||
.await
|
||||
}
|
||||
|
||||
/// Rename a table in the database.
|
||||
///
|
||||
/// This is only supported in LanceDB Cloud.
|
||||
|
||||
@@ -307,10 +307,6 @@ pub trait Database:
|
||||
) -> Result<crate::function::FunctionVersion> {
|
||||
function_catalog_not_supported()
|
||||
}
|
||||
/// Drop one exact immutable Function version from the remote catalog.
|
||||
async fn drop_function(&self, _name: &str, _version: &str) -> Result<bool> {
|
||||
function_catalog_not_supported()
|
||||
}
|
||||
/// A [`crate::job::Job`] handle for a server-side job by id, suitable for
|
||||
/// waiting on or cancelling the job. The handle is constructed without a
|
||||
/// server round trip; an unknown id surfaces when the handle is used.
|
||||
|
||||
@@ -539,7 +539,9 @@ impl Database for LanceNamespaceDatabase {
|
||||
self.namespace
|
||||
.drop_table(drop_request)
|
||||
.await
|
||||
.map_err(|e| map_namespace_lance_error(e, name))?;
|
||||
.map_err(|e| Error::Runtime {
|
||||
message: format!("Failed to drop table: {}", e),
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1493,15 +1495,6 @@ mod tests {
|
||||
.expect("Failed to list tables");
|
||||
assert!(!table_names_after.contains(&"drop_test".to_string()));
|
||||
|
||||
let error = conn
|
||||
.drop_table("drop_test", &["test_ns".into()])
|
||||
.await
|
||||
.expect_err("dropping a missing table should fail");
|
||||
assert!(
|
||||
matches!(error, Error::TableNotFound { ref name, .. } if name == "drop_test"),
|
||||
"expected TableNotFound, got: {error:?}"
|
||||
);
|
||||
|
||||
// Verify: Cannot open dropped table
|
||||
let open_result = conn.open_table("drop_test").execute().await;
|
||||
assert!(open_result.is_err());
|
||||
|
||||
@@ -15,9 +15,6 @@ use serde_json::Value;
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
/// Semantic Function type for a Blob v2 value.
|
||||
pub const FUNCTION_BLOB_V2_TYPE: &str = "blob_v2";
|
||||
|
||||
fn invalid_json(error: impl std::fmt::Display) -> Error {
|
||||
Error::InvalidInput {
|
||||
message: format!("invalid remote Function JSON: {error}"),
|
||||
|
||||
@@ -533,11 +533,6 @@ struct RemoteListJobsResponse {
|
||||
page_token: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct RemoteDropFunctionResponse {
|
||||
dropped: bool,
|
||||
}
|
||||
|
||||
/// Bound on `list_jobs` page walking; a warning is logged when the listing
|
||||
/// is truncated at this many pages.
|
||||
const MAX_LIST_JOBS_PAGES: usize = 100;
|
||||
@@ -588,20 +583,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
response.json().await.err_to_http(request_id)
|
||||
}
|
||||
|
||||
async fn drop_function(&self, name: &str, version: &str) -> Result<bool> {
|
||||
let req = self
|
||||
.client
|
||||
.post("/v1/functions/drop")
|
||||
.json(&serde_json::json!({
|
||||
"name": name,
|
||||
"version": version,
|
||||
}));
|
||||
let (request_id, response) = self.client.send(req).await?;
|
||||
let response = self.client.check_response(&request_id, response).await?;
|
||||
let response: RemoteDropFunctionResponse = response.json().await.err_to_http(request_id)?;
|
||||
Ok(response.dropped)
|
||||
}
|
||||
|
||||
fn job(&self, job_id: &str) -> Result<crate::job::Job> {
|
||||
Ok(crate::job::Job::new(Box::new(super::job::RemoteJob::new(
|
||||
self.client.clone(),
|
||||
@@ -2708,25 +2689,6 @@ mod tests {
|
||||
assert_eq!(version.version(), "fv_01K3EXACT");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_drop_function_sends_exact_version_and_decodes_replay() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||
assert_eq!(request.url().path(), "/v1/functions/drop");
|
||||
let body: serde_json::Value =
|
||||
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
|
||||
assert_eq!(
|
||||
body,
|
||||
serde_json::json!({"name": "embed", "version": "fv_01K3EXACT"})
|
||||
);
|
||||
http::Response::builder()
|
||||
.status(200)
|
||||
.body(r#"{"dropped":false}"#)
|
||||
.unwrap()
|
||||
});
|
||||
assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_conn_job_waits_to_done() {
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -29,16 +29,14 @@ use datafusion_common::{ScalarValue, tree_node::TreeNode};
|
||||
use datafusion_expr::Expr;
|
||||
use datafusion_physical_plan::PhysicalExpr;
|
||||
use lance::dataset::NewColumnTransform;
|
||||
use lance_arrow::{ARROW_EXT_NAME_KEY, BLOB_V2_EXT_NAME, FieldExt};
|
||||
use lance_core::datatypes::{
|
||||
BLOB_V2_DESC_FIELD, BlobV2Layout, format_field_path_minimal, parse_field_path,
|
||||
};
|
||||
use lance_arrow::FieldExt;
|
||||
use lance_core::datatypes::{BLOB_V2_DESC_FIELD, format_field_path_minimal, parse_field_path};
|
||||
use lance_datafusion::planner::Planner;
|
||||
use lance_namespace::models::{JsonArrowDataType, JsonArrowField, JsonArrowSchema};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::function::{FUNCTION_BLOB_V2_TYPE, FunctionApplication, FunctionBinding};
|
||||
use crate::function::{FunctionApplication, FunctionBinding};
|
||||
use crate::utils::resolve_arrow_field_path;
|
||||
use crate::{Error, Result};
|
||||
|
||||
@@ -583,23 +581,6 @@ fn resolve_field_path<'a>(schema: &'a ArrowSchema, path: &str) -> Result<Resolve
|
||||
}
|
||||
|
||||
fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
|
||||
let is_blob_v2 = field
|
||||
.metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.get(ARROW_EXT_NAME_KEY))
|
||||
.map(String::as_str)
|
||||
== Some(BLOB_V2_EXT_NAME);
|
||||
if is_blob_v2 {
|
||||
let arrow_field = lance_namespace::schema::convert_json_arrow_field(field)
|
||||
.map_err(|e| invalid_function(format!("invalid Function input field: {e}")))?;
|
||||
if !has_supported_blob_v2_layout(&arrow_field) {
|
||||
return Err(invalid_function(format!(
|
||||
"Function input '{}' has an invalid Blob v2 storage layout",
|
||||
arrow_field.name()
|
||||
)));
|
||||
}
|
||||
return Ok(FUNCTION_BLOB_V2_TYPE.to_string());
|
||||
}
|
||||
if field.r#type.fields.is_none() && field.r#type.length.is_none() {
|
||||
Ok(field.r#type.r#type.clone())
|
||||
} else {
|
||||
@@ -609,14 +590,6 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result<String> {
|
||||
}
|
||||
}
|
||||
|
||||
fn has_supported_blob_v2_layout(field: &ArrowField) -> bool {
|
||||
field.is_blob_v2()
|
||||
&& matches!(
|
||||
field.data_type(),
|
||||
DataType::Struct(fields) if BlobV2Layout::classify(fields).is_some()
|
||||
)
|
||||
}
|
||||
|
||||
/// `fixed_size_list<item, size>` -> (`item`, `size`); the comma must sit outside
|
||||
/// any nested `<...>`.
|
||||
fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> {
|
||||
@@ -696,76 +669,6 @@ fn parse_output_arrow_type(raw: &str) -> Result<JsonArrowDataType> {
|
||||
Ok(data_type)
|
||||
}
|
||||
|
||||
fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonArrowField> {
|
||||
if raw == FUNCTION_BLOB_V2_TYPE {
|
||||
return lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
|
||||
crate::blob(name, nullable),
|
||||
]))
|
||||
.map_err(|e| invalid_function(format!("could not encode Blob v2 output field: {e}")))?
|
||||
.fields
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(|| invalid_function("Blob v2 output field is missing"));
|
||||
}
|
||||
Ok(JsonArrowField::new(
|
||||
name.to_string(),
|
||||
nullable,
|
||||
parse_output_arrow_type(raw)?,
|
||||
))
|
||||
}
|
||||
|
||||
fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
|
||||
expected.name() == actual.name()
|
||||
&& expected.is_nullable() == actual.is_nullable()
|
||||
&& if expected.is_blob_v2() {
|
||||
has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
|
||||
} else {
|
||||
function_output_type_matches(expected.data_type(), actual.data_type())
|
||||
}
|
||||
}
|
||||
|
||||
fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
|
||||
if expected == actual {
|
||||
return true;
|
||||
}
|
||||
match (expected, actual) {
|
||||
(DataType::Struct(expected), DataType::Struct(actual)) => {
|
||||
expected.len() == actual.len()
|
||||
&& expected
|
||||
.iter()
|
||||
.zip(actual)
|
||||
.all(|(expected, actual)| function_output_field_matches(expected, actual))
|
||||
}
|
||||
(DataType::List(expected), DataType::List(actual))
|
||||
| (DataType::LargeList(expected), DataType::LargeList(actual)) => {
|
||||
function_output_field_matches(expected, actual)
|
||||
}
|
||||
(
|
||||
DataType::FixedSizeList(expected, expected_size),
|
||||
DataType::FixedSizeList(actual, actual_size),
|
||||
) => expected_size == actual_size && function_output_field_matches(expected, actual),
|
||||
(DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
|
||||
expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn function_output_type_has_blob(data_type: &DataType) -> bool {
|
||||
match data_type {
|
||||
DataType::Struct(fields) => fields
|
||||
.iter()
|
||||
.any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
|
||||
DataType::List(field)
|
||||
| DataType::LargeList(field)
|
||||
| DataType::FixedSizeList(field, _)
|
||||
| DataType::Map(field, _) => {
|
||||
field.is_blob_v2() || function_output_type_has_blob(field.data_type())
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding) -> Result<()> {
|
||||
let mut input_fields = Vec::with_capacity(binding.inputs().len());
|
||||
for input in binding.inputs() {
|
||||
@@ -846,18 +749,10 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
|
||||
(has_supported_blob_v2_layout(field), true)
|
||||
} else {
|
||||
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
|
||||
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
|
||||
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
|
||||
(
|
||||
function_output_type_matches(&expected_type, field.data_type()),
|
||||
function_output_type_has_blob(&expected_type),
|
||||
)
|
||||
};
|
||||
if !type_matches {
|
||||
let expected_type = parse_output_arrow_type(&output.arrow_type)?;
|
||||
let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
|
||||
.map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
|
||||
if field.data_type() != &expected_type {
|
||||
return Err(invalid_function(format!(
|
||||
"Function output '{}' type no longer matches binding '{}'",
|
||||
output.output_name,
|
||||
@@ -886,21 +781,15 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
|
||||
binding.binding_id()
|
||||
)));
|
||||
}
|
||||
if has_semantic_blob {
|
||||
output_fields.push(function_output_field(
|
||||
field.name(),
|
||||
true,
|
||||
&output.arrow_type,
|
||||
)?);
|
||||
} else {
|
||||
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
|
||||
ArrowField::new(field.name().clone(), field.data_type().clone(), true),
|
||||
]))
|
||||
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
|
||||
output_fields.push(json.fields.into_iter().next().unwrap());
|
||||
}
|
||||
output_fields.push(ArrowField::new(
|
||||
field.name().clone(),
|
||||
field.data_type().clone(),
|
||||
true,
|
||||
));
|
||||
}
|
||||
let output_schema = JsonArrowSchema::new(output_fields);
|
||||
let output_schema =
|
||||
lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(output_fields))
|
||||
.map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
|
||||
let output_schema = serde_json::to_value(output_schema).map_err(|e| {
|
||||
invalid_function(format!(
|
||||
"could not encode exact Function output schema: {e}"
|
||||
@@ -1029,15 +918,16 @@ pub(crate) fn plan_function_application(
|
||||
"Function logical outputs must be non-nullable during NULL assignment",
|
||||
));
|
||||
}
|
||||
let arrow_type = output.arrow_type.as_deref().ok_or_else(|| {
|
||||
invalid_function("scalar Function output is missing its Arrow type")
|
||||
})?;
|
||||
let data_type =
|
||||
parse_output_arrow_type(output.arrow_type.as_deref().ok_or_else(|| {
|
||||
invalid_function("scalar Function output is missing its Arrow type")
|
||||
})?)?;
|
||||
outputs.push(FunctionOutputTarget {
|
||||
result_field: WHOLE_RESULT_FIELD.to_string(),
|
||||
output_name: name.to_string(),
|
||||
output_ordinal: 0,
|
||||
});
|
||||
output_fields.push(function_output_field(name, true, arrow_type)?);
|
||||
output_fields.push(JsonArrowField::new(name.to_string(), true, data_type));
|
||||
}
|
||||
"named_struct" => {
|
||||
if output.fields.is_empty() {
|
||||
@@ -1081,7 +971,13 @@ pub(crate) fn plan_function_application(
|
||||
let fields = output
|
||||
.fields
|
||||
.iter()
|
||||
.map(|field| function_output_field(&field.name, false, &field.arrow_type))
|
||||
.map(|field| {
|
||||
Ok(JsonArrowField::new(
|
||||
field.name.clone(),
|
||||
false,
|
||||
parse_output_arrow_type(&field.arrow_type)?,
|
||||
))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let mut data_type = JsonArrowDataType::new("struct".to_string());
|
||||
data_type.fields = Some(fields);
|
||||
@@ -1108,7 +1004,11 @@ pub(crate) fn plan_function_application(
|
||||
output_name: name.clone(),
|
||||
output_ordinal: ordinal as u32,
|
||||
});
|
||||
output_fields.push(function_output_field(name, true, &field.arrow_type)?);
|
||||
output_fields.push(JsonArrowField::new(
|
||||
name.clone(),
|
||||
true,
|
||||
parse_output_arrow_type(&field.arrow_type)?,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1745,7 +1645,7 @@ mod tests {
|
||||
}
|
||||
|
||||
use arrow_array::record_batch;
|
||||
use arrow_schema::{DataType, TimeUnit};
|
||||
use arrow_schema::DataType;
|
||||
use futures::TryStreamExt;
|
||||
use lance::dataset::ColumnAlteration;
|
||||
|
||||
@@ -2706,73 +2606,6 @@ mod tests {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn blob_application(output: &str) -> FunctionApplication {
|
||||
FunctionApplication::from_json(&format!(
|
||||
r#"{{
|
||||
"function":{{"name":"blob_features","version":"fv_blob"}},
|
||||
"inputs":[
|
||||
{{"parameter":"image","kind":"column","value":{{"path":"image"}}}}
|
||||
],
|
||||
"output":{output}
|
||||
}}"#
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn binding_from_plan(plan: &FunctionDeclarationPlan) -> FunctionBinding {
|
||||
let inputs = plan
|
||||
.input_bindings
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, input)| {
|
||||
serde_json::json!({
|
||||
"parameter": input.parameter,
|
||||
"field_id": index,
|
||||
"field_path": input.field_path,
|
||||
"arrow_type": input.arrow_type,
|
||||
"nullable": input.nullable,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let outputs = plan
|
||||
.outputs
|
||||
.iter()
|
||||
.zip(&plan.output_schema.fields)
|
||||
.enumerate()
|
||||
.map(|(index, (output, field))| {
|
||||
serde_json::json!({
|
||||
"result_field": output.result_field,
|
||||
"output_name": output.output_name,
|
||||
"output_field_id": 100 + index,
|
||||
"output_ordinal": output.output_ordinal,
|
||||
"arrow_type": canonical_input_arrow_type(field).unwrap(),
|
||||
"nullable": false,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
FunctionBinding::from_json(
|
||||
&serde_json::json!({
|
||||
"binding_id": "fb_blob",
|
||||
"function": plan.application.function(),
|
||||
"inputs": inputs,
|
||||
"outputs": outputs,
|
||||
"input_schema": plan.input_schema,
|
||||
"output_schema": plan.output_schema,
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn full_blob_field(name: &str, nullable: bool) -> ArrowField {
|
||||
ArrowField::new(
|
||||
name,
|
||||
DataType::Struct(lance_core::datatypes::BLOB_V2_LOGICAL_FIELDS.clone()),
|
||||
nullable,
|
||||
)
|
||||
.with_metadata(crate::blob(name, nullable).metadata().clone())
|
||||
}
|
||||
|
||||
fn function_binding_schema(title_nullable: bool, body_nullable: bool) -> ArrowSchema {
|
||||
ArrowSchema::new(vec![
|
||||
ArrowField::new("title", DataType::Utf8, title_nullable),
|
||||
@@ -3046,151 +2879,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_function_plans_semantic_input_and_scalar_output() {
|
||||
let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
|
||||
let application =
|
||||
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
|
||||
let plan = plan_function_application(&schema, &application, Some("thumbnail")).unwrap();
|
||||
|
||||
assert_eq!(plan.input_bindings[0].arrow_type, FUNCTION_BLOB_V2_TYPE);
|
||||
let input_schema =
|
||||
lance_namespace::schema::convert_json_arrow_schema(&plan.input_schema).unwrap();
|
||||
assert!(input_schema.field(0).is_blob_v2());
|
||||
let output_schema =
|
||||
lance_namespace::schema::convert_json_arrow_schema(&plan.output_schema).unwrap();
|
||||
assert!(output_schema.field(0).is_blob_v2());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_scalar_binding_accepts_full_logical_layout() {
|
||||
let input = crate::blob("image", false);
|
||||
let application =
|
||||
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
|
||||
let plan = plan_function_application(
|
||||
&ArrowSchema::new(vec![input.clone()]),
|
||||
&application,
|
||||
Some("thumbnail"),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = binding_from_plan(&plan);
|
||||
let mut metadata = full_blob_field("thumbnail", true).metadata().clone();
|
||||
metadata.extend(function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
0,
|
||||
&["image".into()],
|
||||
));
|
||||
let output = full_blob_field("thumbnail", true).with_metadata(metadata);
|
||||
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, output]), &binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_binding_rejects_marker_on_invalid_storage_layout() {
|
||||
let input = crate::blob("image", false);
|
||||
let application =
|
||||
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
|
||||
let plan = plan_function_application(
|
||||
&ArrowSchema::new(vec![input.clone()]),
|
||||
&application,
|
||||
Some("thumbnail"),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = binding_from_plan(&plan);
|
||||
let malformed = ArrowField::new("thumbnail", DataType::Int64, true)
|
||||
.with_metadata(crate::blob("thumbnail", true).metadata().clone());
|
||||
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, malformed]), &binding)
|
||||
.unwrap_err();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_input_rejects_marker_on_invalid_storage_layout() {
|
||||
let malformed = ArrowField::new("image", DataType::Int64, false)
|
||||
.with_metadata(crate::blob("image", false).metadata().clone());
|
||||
let application =
|
||||
blob_application(r#"{"kind":"scalar","arrow_type":"blob_v2","nullable":false}"#);
|
||||
|
||||
plan_function_application(
|
||||
&ArrowSchema::new(vec![malformed]),
|
||||
&application,
|
||||
Some("thumbnail"),
|
||||
)
|
||||
.unwrap_err();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_blob_input_does_not_require_json_round_trip() {
|
||||
let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
|
||||
ArrowField::new("event_time", DataType::Time64(TimeUnit::Microsecond), false),
|
||||
]))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
canonical_input_arrow_type(&json.fields[0]).unwrap(),
|
||||
"time64"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_named_struct_plans_expanded_and_whole_outputs() {
|
||||
let schema = ArrowSchema::new(vec![crate::blob("image", false)]);
|
||||
let application = blob_application(
|
||||
r#"{"kind":"named_struct","fields":[
|
||||
{"name":"thumbnail","arrow_type":"blob_v2","nullable":false},
|
||||
{"name":"width","arrow_type":"int32","nullable":false}
|
||||
]}"#,
|
||||
);
|
||||
|
||||
let expanded = plan_function_application(&schema, &application, None).unwrap();
|
||||
let expanded_schema =
|
||||
lance_namespace::schema::convert_json_arrow_schema(&expanded.output_schema).unwrap();
|
||||
assert!(expanded_schema.field(0).is_blob_v2());
|
||||
assert_eq!(expanded_schema.field(1).data_type(), &DataType::Int32);
|
||||
|
||||
let whole = plan_function_application(&schema, &application, Some("analysis")).unwrap();
|
||||
let whole_schema =
|
||||
lance_namespace::schema::convert_json_arrow_schema(&whole.output_schema).unwrap();
|
||||
let DataType::Struct(fields) = whole_schema.field(0).data_type() else {
|
||||
panic!("whole Function output should be a struct");
|
||||
};
|
||||
assert!(fields[0].is_blob_v2());
|
||||
assert_eq!(fields[1].data_type(), &DataType::Int32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blob_whole_struct_binding_accepts_full_logical_layout() {
|
||||
let input = crate::blob("image", false);
|
||||
let application = blob_application(
|
||||
r#"{"kind":"named_struct","fields":[
|
||||
{"name":"thumbnail","arrow_type":"blob_v2","nullable":false},
|
||||
{"name":"width","arrow_type":"int32","nullable":false}
|
||||
]}"#,
|
||||
);
|
||||
let plan = plan_function_application(
|
||||
&ArrowSchema::new(vec![input.clone()]),
|
||||
&application,
|
||||
Some("analysis"),
|
||||
)
|
||||
.unwrap();
|
||||
let binding = binding_from_plan(&plan);
|
||||
|
||||
let output = ArrowField::new(
|
||||
"analysis",
|
||||
DataType::Struct(Fields::from(vec![
|
||||
full_blob_field("thumbnail", false),
|
||||
ArrowField::new("width", DataType::Int32, false),
|
||||
])),
|
||||
true,
|
||||
)
|
||||
.with_metadata(function_computed_column_metadata(
|
||||
binding.binding_id(),
|
||||
0,
|
||||
&["image".into()],
|
||||
));
|
||||
ensure_binding_matches_schema(&ArrowSchema::new(vec![input, output]), &binding).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_function_mapping_and_sibling_collisions_fail_before_request() {
|
||||
let unknown = named_struct_application(r#"{"missing":"renamed"}"#);
|
||||
|
||||
@@ -45,11 +45,7 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
|
||||
.get_function("normalize_score", "fv_exact")
|
||||
.await
|
||||
.unwrap_err();
|
||||
let drop_error = connection
|
||||
.drop_function("normalize_score", "fv_exact")
|
||||
.await
|
||||
.unwrap_err();
|
||||
for error in [create_error, lookup_error, drop_error] {
|
||||
for error in [create_error, lookup_error] {
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::NotSupported { message }
|
||||
|
||||
@@ -78,12 +78,6 @@
|
||||
"type": "utf8"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "large_utf8",
|
||||
"json": {
|
||||
"type": "large_utf8"
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "binary",
|
||||
"json": {
|
||||
@@ -177,21 +171,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "list<large_utf8>",
|
||||
"json": {
|
||||
"type": "list",
|
||||
"fields": [
|
||||
{
|
||||
"name": "item",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": "large_utf8"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "large_list<utf8>",
|
||||
"json": {
|
||||
@@ -207,21 +186,6 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "large_list<large_utf8>",
|
||||
"json": {
|
||||
"type": "large_list",
|
||||
"fields": [
|
||||
{
|
||||
"name": "item",
|
||||
"nullable": false,
|
||||
"type": {
|
||||
"type": "large_utf8"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"arrow_type": "fixed_size_list<float32, 384>",
|
||||
"json": {
|
||||
@@ -366,4 +330,4 @@
|
||||
"timestamp[us]",
|
||||
"struct<a: int32>"
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user