feat(python): support exist_ok for Function creation (#4992)

Adds `exist_ok=False` to Python Function creation and job submission.
`exist_ok=True` sends `CREATE FUNCTION IF NOT EXISTS` and returns the
stored immutable version, preserving its code, credentials and execution
settings. The default retains the current version-creation behavior.

```python
version = db.create_function(analyze_caption, exist_ok=True)
```

The option also exists on the Rust registration request and is kept out
of the canonical Function definition payload. Includes
sync/async/background protocol and stored-version coverage. Requires
matching SQL server support.

BREAKING CHANGE: Rust `FunctionRegistrationRequest` struct literals must
initialize `exist_ok`; legacy serialized requests default to false.

No local builds or tests were run, as requested.
This commit is contained in:
Jack Ye
2026-11-14 10:35:54 +08:00
committed by GitHub
parent b061c3627c
commit 2c3c30dbdf
10 changed files with 155 additions and 11 deletions
+13
View File
@@ -111,6 +111,19 @@ with `db.get_function(name, version=...)`. The version may be an integer or
its string form. Omit `version` to resolve and return the latest currently
visible revision.
{/* Draft for the function exist_ok release.
Use `exist_ok=True` on `create_function` or `create_function_async` to return
the existing named version without changing its code, credentials or settings.
This requires a deployment that supports the option. The default retains the
current behavior of registering a new version.
```python
version = db.create_function(embed, exist_ok=True)
```
*/}
<Note>
The SQL command submits an internal `create_function` image-build job and
waits for publication. The SDK returns the SQL query handle. Query handles
+9
View File
@@ -16,6 +16,15 @@ availability:
## CREATE FUNCTION {#create}
{/* Draft for the function exist_ok release.
`CREATE FUNCTION IF NOT EXISTS` preserves an existing named Function version,
including its code, credentials and execution settings. In Python, pass
`exist_ok=True` to `create_function` or `create_function_async`. The default
retains the existing version-creation behavior.
*/}
<CodeGroup>
```sql SQL icon="database"
CREATE FUNCTION f(x int64) RETURNS int64
+5 -1
View File
@@ -188,7 +188,11 @@ class Connection(object):
async def server_info(self) -> str: ...
async def list_gpu_classes(self) -> list[GpuClass]: ...
async def create_function_async(
self, request_json: str, namespace_path: Optional[List[str]] = None
self,
request_json: str,
namespace_path: Optional[List[str]] = None,
*,
exist_ok: bool = False,
) -> Job: ...
async def get_function(
self,
+22 -3
View File
@@ -1004,10 +1004,14 @@ class DBConnection(EnforceOverrides):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> FunctionVersion:
"""Build and register a scalar Python function, then return its version.
``exist_ok=True`` returns the existing named Function version unchanged.
By default, registration retains its existing version-creation behavior.
The server builds the OCI image and registers the completed artifact.
This is the blocking counterpart of :meth:`create_function_async`.
LanceDB Embedded connections raise ``NotImplementedError``.
@@ -1040,7 +1044,10 @@ class DBConnection(EnforceOverrides):
```
"""
return self.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
).wait()
def create_function_async(
@@ -1048,10 +1055,14 @@ class DBConnection(EnforceOverrides):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
"""Submit a scalar Python function for building and registration.
``exist_ok=True`` preserves an existing named version and its metadata.
The default retains the existing version-creation behavior.
The server-side job builds the OCI image, then registers the completed
artifact. The returned job has type ``query`` and the SQL query UUID as
its ID. Waiting on it returns the immutable Function version.
@@ -2396,11 +2407,15 @@ class LanceDBConnection(DBConnection):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
return Job(job)
@@ -3732,10 +3747,14 @@ class AsyncConnection(object):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> AsyncJob[FunctionVersion]:
"""Submit a scalar Python function for building and registration.
``exist_ok=True`` preserves an existing named version and its metadata.
The default retains the existing version-creation behavior.
The server-side job builds the OCI image, then registers the completed
artifact. The returned job has type ``query`` and the SQL query UUID as
its ID. Waiting on it returns the immutable Function version.
@@ -3749,7 +3768,7 @@ class AsyncConnection(object):
raise TypeError("create_function_async requires a @function definition")
request = definition.bind_secrets(secrets)
inner = await self._inner.create_function_async(
request.to_canonical_json(), namespace_path
request.to_canonical_json(), namespace_path, exist_ok=exist_ok
)
return _typed_job(inner, FunctionVersion.from_json)
+5 -1
View File
@@ -992,11 +992,15 @@ class RemoteDBConnection(DBConnection):
definition: FunctionDefinition,
*,
secrets: Optional[Sequence[EnvVarSecret]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> Job[FunctionVersion]:
job = LOOP.run(
self._conn.create_function_async(
definition, secrets=secrets, namespace_path=namespace_path
definition,
secrets=secrets,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
return Job(job)
@@ -0,0 +1,63 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import json
from pathlib import Path
import pyarrow as pa
import pytest
import lancedb
from lancedb.functions import function
@pytest.mark.asyncio
@pytest.mark.parametrize("entry", ["sync", "sync_job", "async_job"])
@pytest.mark.parametrize("exist_ok", [None, False, True])
async def test_function_exist_ok_forwards_sql_and_keeps_stored_version(entry, exist_ok):
flight = pytest.importorskip("pyarrow.flight")
fixture = (
Path(__file__).parents[3]
/ "rust/lancedb/tests/fixtures/first_class_functions/v1"
/ "remote_function_version.canonical.json"
)
stored = json.loads(fixture.read_text())
result = pa.table({"definition": [json.dumps(stored)]})
@function(name=stored["name"])
def replacement(value: float) -> float:
return value * 2
class FunctionServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.requests = []
def do_get(self, context, ticket):
self.requests.append(ticket.ticket)
return flight.RecordBatchStream(result)
with FunctionServer() as server:
options = {
"api_key": "test-key",
"host_override": "http://localhost:10024",
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
}
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
kwargs["namespace_path"] = ["team"]
if entry == "async_job":
db = await lancedb.connect_async("db://analytics", **options)
job = await db.create_function_async(replacement, **kwargs)
version = await job.wait()
else:
db = lancedb.connect("db://analytics", **options)
if entry == "sync_job":
version = db.create_function_async(replacement, **kwargs).wait()
else:
version = db.create_function(replacement, **kwargs)
(request,) = server.requests
prefix = "CREATE FUNCTION IF NOT EXISTS" if exist_ok else "CREATE FUNCTION"
assert f'{prefix} "{stored["name"]}" FROM '.encode() in request
assert b'"exist_ok"' not in request
assert json.loads(version.to_canonical_json()) == stored
+4 -2
View File
@@ -903,15 +903,17 @@ impl Connection {
})
}
#[pyo3(signature = (request_json, namespace_path=None))]
#[pyo3(signature = (request_json, namespace_path=None, *, exist_ok=false))]
pub fn create_function_async(
self_: PyRef<'_, Self>,
request_json: String,
namespace_path: Option<Vec<String>>,
exist_ok: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
let mut request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
.infer_error()?;
request.exist_ok = exist_ok;
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner
+7
View File
@@ -834,6 +834,9 @@ fn validate_row_timeout_value(value: Option<f64>) -> std::result::Result<(), &'s
/// database already holds, which Sophon resolves inside the remote runtime.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct FunctionRegistrationRequest {
/// Preserve an existing named Function version instead of registering a new one.
#[serde(default, skip_serializing_if = "is_false")]
pub exist_ok: bool,
/// Default retries of failed user-code calls. Enterprise only.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub retry_policy: Option<FunctionRetryPolicy>,
@@ -876,6 +879,10 @@ pub struct FunctionRegistrationRequest {
pub models: Vec<HuggingFaceModel>,
}
fn is_false(value: &bool) -> bool {
!*value
}
impl FunctionRegistrationRequest {
/// Decode a registration request supplied by an SDK language binding.
pub fn from_json(json: &str) -> Result<Self> {
+10 -1
View File
@@ -34,13 +34,22 @@ impl SqlClient {
validate_function_name(&request.name)?;
validate_namespace(namespace)?;
let name = quote_identifier(&request.name);
let if_not_exists = if request.exist_ok {
" IF NOT EXISTS"
} else {
""
};
let mut body = serde_json::to_value(request).map_err(|error| Error::Runtime {
message: error.to_string(),
})?;
body.as_object_mut().unwrap().remove("name");
body.as_object_mut().unwrap().remove("exist_ok");
self.object_job(
ExecuteQueryRequest {
query: format!("CREATE FUNCTION {name} FROM {}", literal(&body.to_string())),
query: format!(
"CREATE FUNCTION{if_not_exists} {name} FROM {}",
literal(&body.to_string())
),
default_namespace_path: namespace.to_vec(),
parameters: None,
},
+17 -3
View File
@@ -3087,7 +3087,16 @@ async fn function_creation_and_description_use_sql_query_handles() {
const REQUEST: &str = include_str!(
"../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
);
for (defer, with_models) in [(false, false), (false, true), (true, false), (true, true)] {
for (defer, with_models, exist_ok) in [
(false, false, false),
(false, true, false),
(true, false, false),
(true, true, false),
(false, false, true),
(false, true, true),
(true, false, true),
(true, true, true),
] {
let mut request_json: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
request_json["batch_size"] = serde_json::json!(2);
request_json["max_concurrency"] = serde_json::json!(4);
@@ -3113,9 +3122,11 @@ async fn function_creation_and_description_use_sql_query_handles() {
let expected_models = version_json["image"].get("models");
let mut expected_request = request_json.clone();
expected_request.as_object_mut().unwrap().remove("name");
let request =
let mut request =
crate::function::FunctionRegistrationRequest::from_json(&request_json.to_string())
.unwrap();
assert!(!request.exist_ok);
request.exist_ok = exist_ok;
let result = RecordBatch::try_from_iter(vec![(
"definition",
Arc::new(StringArray::from(vec![version_json.to_string()])) as ArrayRef,
@@ -3151,7 +3162,10 @@ async fn function_creation_and_description_use_sql_query_handles() {
{
let seen = statements.lock().unwrap();
let encoded = seen[0]
.strip_prefix(r#"CREATE FUNCTION "normalize_score" FROM '"#)
.strip_prefix(&format!(
"CREATE FUNCTION{} \"normalize_score\" FROM '",
if exist_ok { " IF NOT EXISTS" } else { "" }
))
.unwrap()
.strip_suffix('\'')
.unwrap();