feat(python): support exist_ok for Function creation (#4992)

Adds `exist_ok=False` to Python Function creation and job submission.
`exist_ok=True` sends `CREATE FUNCTION IF NOT EXISTS` and returns the
stored immutable version, preserving its code, credentials and execution
settings. The default retains the current version-creation behavior.

```python
version = db.create_function(analyze_caption, exist_ok=True)
```

The option also exists on the Rust registration request and is kept out
of the canonical Function definition payload. Includes
sync/async/background protocol and stored-version coverage. Requires
matching SQL server support.

BREAKING CHANGE: Rust `FunctionRegistrationRequest` struct literals must
initialize `exist_ok`; legacy serialized requests default to false.

No local builds or tests were run, as requested.
This commit is contained in:
Jack Ye
2026-11-14 10:35:54 +08:00
committed by GitHub
parent b061c3627c
commit 2c3c30dbdf
10 changed files with 155 additions and 11 deletions
+5 -1
View File
@@ -188,7 +188,11 @@ class Connection(object):
async def server_info(self) -> str: ...
async def list_gpu_classes(self) -> list[GpuClass]: ...
async def create_function_async(
self, request_json: str, namespace_path: Optional[List[str]] = None
self,
request_json: str,
namespace_path: Optional[List[str]] = None,
*,
exist_ok: bool = False,
) -> Job: ...
async def get_function(
self,
+22 -3
View File
@@ -1004,10 +1004,14 @@ class DBConnection(EnforceOverrides):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Build and register a scalar Python function, then return its version.
``exist_ok=True`` returns the existing named Function version unchanged.
By default, registration retains its existing version-creation behavior.
The server builds the OCI image and registers the completed artifact.
This is the blocking counterpart of :meth:`create_function_async`.
LanceDB Embedded connections raise ``NotImplementedError``.
@@ -1040,7 +1044,10 @@ class DBConnection(EnforceOverrides):
```
"""
return self.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
).wait()
def create_function_async(
@@ -1048,10 +1055,14 @@ class DBConnection(EnforceOverrides):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
"""Submit a scalar Python function for building and registration.
``exist_ok=True`` preserves an existing named version and its metadata.
The default retains the existing version-creation behavior.
The server-side job builds the OCI image, then registers the completed
artifact. The returned job has type ``query`` and the SQL query UUID as
its ID. Waiting on it returns the immutable Function version.
@@ -2396,11 +2407,15 @@ class LanceDBConnection(DBConnection):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
return Job(job)
@@ -3732,10 +3747,14 @@ class AsyncConnection(object):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> AsyncJob[FunctionVersion]:
"""Submit a scalar Python function for building and registration.
``exist_ok=True`` preserves an existing named version and its metadata.
The default retains the existing version-creation behavior.
The server-side job builds the OCI image, then registers the completed
artifact. The returned job has type ``query`` and the SQL query UUID as
its ID. Waiting on it returns the immutable Function version.
@@ -3749,7 +3768,7 @@ class AsyncConnection(object):
raise TypeError("create_function_async requires a @function definition")
request = definition.bind_secrets(secrets)
inner = await self._inner.create_function_async(
request.to_canonical_json(), namespace_path
request.to_canonical_json(), namespace_path, exist_ok=exist_ok
)
return _typed_job(inner, FunctionVersion.from_json)
+5 -1
View File
@@ -992,11 +992,15 @@ class RemoteDBConnection(DBConnection):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
return Job(job)
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import json
from pathlib import Path
import pyarrow as pa
import pytest
import lancedb
from lancedb.functions import function
@pytest.mark.asyncio
@pytest.mark.parametrize("entry", ["sync", "sync_job", "async_job"])
@pytest.mark.parametrize("exist_ok", [None, False, True])
async def test_function_exist_ok_forwards_sql_and_keeps_stored_version(entry, exist_ok):
flight = pytest.importorskip("pyarrow.flight")
fixture = (
Path(__file__).parents[3]
/ "rust/lancedb/tests/fixtures/first_class_functions/v1"
/ "remote_function_version.canonical.json"
)
stored = json.loads(fixture.read_text())
result = pa.table({"definition": [json.dumps(stored)]})
@function(name=stored["name"])
def replacement(value: float) -> float:
return value * 2
class FunctionServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.requests = []
def do_get(self, context, ticket):
self.requests.append(ticket.ticket)
return flight.RecordBatchStream(result)
with FunctionServer() as server:
options = {
"api_key": "test-key",
"host_override": "http://localhost:10024",
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
}
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
kwargs["namespace_path"] = ["team"]
if entry == "async_job":
db = await lancedb.connect_async("db://analytics", **options)
job = await db.create_function_async(replacement, **kwargs)
version = await job.wait()
else:
db = lancedb.connect("db://analytics", **options)
if entry == "sync_job":
version = db.create_function_async(replacement, **kwargs).wait()
else:
version = db.create_function(replacement, **kwargs)
(request,) = server.requests
prefix = "CREATE FUNCTION IF NOT EXISTS" if exist_ok else "CREATE FUNCTION"
assert f'{prefix} "{stored["name"]}" FROM '.encode() in request
assert b'"exist_ok"' not in request
assert json.loads(version.to_canonical_json()) == stored
+4 -2
View File
@@ -903,15 +903,17 @@ impl Connection {
})
}
#[pyo3(signature = (request_json, namespace_path=None))]
#[pyo3(signature = (request_json, namespace_path=None, *, exist_ok=false))]
pub fn create_function_async(
self_: PyRef<'_, Self>,
request_json: String,
namespace_path: Option<Vec<String>>,
exist_ok: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
let mut request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
.infer_error()?;
request.exist_ok = exist_ok;
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner