mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok for Function creation (#4992)
Adds `exist_ok=False` to Python Function creation and job submission. `exist_ok=True` sends `CREATE FUNCTION IF NOT EXISTS` and returns the stored immutable version, preserving its code, credentials and execution settings. The default retains the current version-creation behavior. ```python version = db.create_function(analyze_caption, exist_ok=True) ``` The option also exists on the Rust registration request and is kept out of the canonical Function definition payload. Includes sync/async/background protocol and stored-version coverage. Requires matching SQL server support. BREAKING CHANGE: Rust `FunctionRegistrationRequest` struct literals must initialize `exist_ok`; legacy serialized requests default to false. No local builds or tests were run, as requested.
This commit is contained in:
@@ -188,7 +188,11 @@ class Connection(object):
|
||||
async def server_info(self) -> str: ...
|
||||
async def list_gpu_classes(self) -> list[GpuClass]: ...
|
||||
async def create_function_async(
|
||||
self, request_json: str, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
request_json: str,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
) -> Job: ...
|
||||
async def get_function(
|
||||
self,
|
||||
|
||||
@@ -1004,10 +1004,14 @@ class DBConnection(EnforceOverrides):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> FunctionVersion:
|
||||
"""Build and register a scalar Python function, then return its version.
|
||||
|
||||
``exist_ok=True`` returns the existing named Function version unchanged.
|
||||
By default, registration retains its existing version-creation behavior.
|
||||
|
||||
The server builds the OCI image and registers the completed artifact.
|
||||
This is the blocking counterpart of :meth:`create_function_async`.
|
||||
LanceDB Embedded connections raise ``NotImplementedError``.
|
||||
@@ -1040,7 +1044,10 @@ class DBConnection(EnforceOverrides):
|
||||
```
|
||||
"""
|
||||
return self.create_function_async(
|
||||
definition, secrets=secrets, namespace_path=namespace_path
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
).wait()
|
||||
|
||||
def create_function_async(
|
||||
@@ -1048,10 +1055,14 @@ class DBConnection(EnforceOverrides):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
"""Submit a scalar Python function for building and registration.
|
||||
|
||||
``exist_ok=True`` preserves an existing named version and its metadata.
|
||||
The default retains the existing version-creation behavior.
|
||||
|
||||
The server-side job builds the OCI image, then registers the completed
|
||||
artifact. The returned job has type ``query`` and the SQL query UUID as
|
||||
its ID. Waiting on it returns the immutable Function version.
|
||||
@@ -2396,11 +2407,15 @@ class LanceDBConnection(DBConnection):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(
|
||||
self._conn.create_function_async(
|
||||
definition, secrets=secrets, namespace_path=namespace_path
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
@@ -3732,10 +3747,14 @@ class AsyncConnection(object):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> AsyncJob[FunctionVersion]:
|
||||
"""Submit a scalar Python function for building and registration.
|
||||
|
||||
``exist_ok=True`` preserves an existing named version and its metadata.
|
||||
The default retains the existing version-creation behavior.
|
||||
|
||||
The server-side job builds the OCI image, then registers the completed
|
||||
artifact. The returned job has type ``query`` and the SQL query UUID as
|
||||
its ID. Waiting on it returns the immutable Function version.
|
||||
@@ -3749,7 +3768,7 @@ class AsyncConnection(object):
|
||||
raise TypeError("create_function_async requires a @function definition")
|
||||
request = definition.bind_secrets(secrets)
|
||||
inner = await self._inner.create_function_async(
|
||||
request.to_canonical_json(), namespace_path
|
||||
request.to_canonical_json(), namespace_path, exist_ok=exist_ok
|
||||
)
|
||||
return _typed_job(inner, FunctionVersion.from_json)
|
||||
|
||||
|
||||
@@ -992,11 +992,15 @@ class RemoteDBConnection(DBConnection):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(
|
||||
self._conn.create_function_async(
|
||||
definition, secrets=secrets, namespace_path=namespace_path
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb.functions import function
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("entry", ["sync", "sync_job", "async_job"])
|
||||
@pytest.mark.parametrize("exist_ok", [None, False, True])
|
||||
async def test_function_exist_ok_forwards_sql_and_keeps_stored_version(entry, exist_ok):
|
||||
flight = pytest.importorskip("pyarrow.flight")
|
||||
fixture = (
|
||||
Path(__file__).parents[3]
|
||||
/ "rust/lancedb/tests/fixtures/first_class_functions/v1"
|
||||
/ "remote_function_version.canonical.json"
|
||||
)
|
||||
stored = json.loads(fixture.read_text())
|
||||
result = pa.table({"definition": [json.dumps(stored)]})
|
||||
|
||||
@function(name=stored["name"])
|
||||
def replacement(value: float) -> float:
|
||||
return value * 2
|
||||
|
||||
class FunctionServer(flight.FlightServerBase):
|
||||
def __init__(self):
|
||||
super().__init__("grpc://127.0.0.1:0")
|
||||
self.requests = []
|
||||
|
||||
def do_get(self, context, ticket):
|
||||
self.requests.append(ticket.ticket)
|
||||
return flight.RecordBatchStream(result)
|
||||
|
||||
with FunctionServer() as server:
|
||||
options = {
|
||||
"api_key": "test-key",
|
||||
"host_override": "http://localhost:10024",
|
||||
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
|
||||
}
|
||||
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
|
||||
kwargs["namespace_path"] = ["team"]
|
||||
if entry == "async_job":
|
||||
db = await lancedb.connect_async("db://analytics", **options)
|
||||
job = await db.create_function_async(replacement, **kwargs)
|
||||
version = await job.wait()
|
||||
else:
|
||||
db = lancedb.connect("db://analytics", **options)
|
||||
if entry == "sync_job":
|
||||
version = db.create_function_async(replacement, **kwargs).wait()
|
||||
else:
|
||||
version = db.create_function(replacement, **kwargs)
|
||||
(request,) = server.requests
|
||||
|
||||
prefix = "CREATE FUNCTION IF NOT EXISTS" if exist_ok else "CREATE FUNCTION"
|
||||
assert f'{prefix} "{stored["name"]}" FROM '.encode() in request
|
||||
assert b'"exist_ok"' not in request
|
||||
assert json.loads(version.to_canonical_json()) == stored
|
||||
@@ -903,15 +903,17 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (request_json, namespace_path=None))]
|
||||
#[pyo3(signature = (request_json, namespace_path=None, *, exist_ok=false))]
|
||||
pub fn create_function_async(
|
||||
self_: PyRef<'_, Self>,
|
||||
request_json: String,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
|
||||
let mut request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
|
||||
.infer_error()?;
|
||||
request.exist_ok = exist_ok;
|
||||
let namespace_path = namespace_path.unwrap_or_default();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner
|
||||
|
||||
Reference in New Issue
Block a user