mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok for Function creation (#4992)
Adds `exist_ok=False` to Python Function creation and job submission. `exist_ok=True` sends `CREATE FUNCTION IF NOT EXISTS` and returns the stored immutable version, preserving its code, credentials and execution settings. The default retains the current version-creation behavior. ```python version = db.create_function(analyze_caption, exist_ok=True) ``` The option also exists on the Rust registration request and is kept out of the canonical Function definition payload. Includes sync/async/background protocol and stored-version coverage. Requires matching SQL server support. BREAKING CHANGE: Rust `FunctionRegistrationRequest` struct literals must initialize `exist_ok`; legacy serialized requests default to false. No local builds or tests were run, as requested.
This commit is contained in:
@@ -111,6 +111,19 @@ with `db.get_function(name, version=...)`. The version may be an integer or
|
||||
its string form. Omit `version` to resolve and return the latest currently
|
||||
visible revision.
|
||||
|
||||
{/* Draft for the function exist_ok release.
|
||||
|
||||
Use `exist_ok=True` on `create_function` or `create_function_async` to return
|
||||
the existing named version without changing its code, credentials or settings.
|
||||
This requires a deployment that supports the option. The default retains the
|
||||
current behavior of registering a new version.
|
||||
|
||||
```python
|
||||
version = db.create_function(embed, exist_ok=True)
|
||||
```
|
||||
|
||||
*/}
|
||||
|
||||
<Note>
|
||||
The SQL command submits an internal `create_function` image-build job and
|
||||
waits for publication. The SDK returns the SQL query handle. Query handles
|
||||
|
||||
@@ -16,6 +16,15 @@ availability:
|
||||
|
||||
## CREATE FUNCTION {#create}
|
||||
|
||||
{/* Draft for the function exist_ok release.
|
||||
|
||||
`CREATE FUNCTION IF NOT EXISTS` preserves an existing named Function version,
|
||||
including its code, credentials and execution settings. In Python, pass
|
||||
`exist_ok=True` to `create_function` or `create_function_async`. The default
|
||||
retains the existing version-creation behavior.
|
||||
|
||||
*/}
|
||||
|
||||
<CodeGroup>
|
||||
```sql SQL icon="database"
|
||||
CREATE FUNCTION f(x int64) RETURNS int64
|
||||
|
||||
@@ -188,7 +188,11 @@ class Connection(object):
|
||||
async def server_info(self) -> str: ...
|
||||
async def list_gpu_classes(self) -> list[GpuClass]: ...
|
||||
async def create_function_async(
|
||||
self, request_json: str, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
request_json: str,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
) -> Job: ...
|
||||
async def get_function(
|
||||
self,
|
||||
|
||||
@@ -1004,10 +1004,14 @@ class DBConnection(EnforceOverrides):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> FunctionVersion:
|
||||
"""Build and register a scalar Python function, then return its version.
|
||||
|
||||
``exist_ok=True`` returns the existing named Function version unchanged.
|
||||
By default, registration retains its existing version-creation behavior.
|
||||
|
||||
The server builds the OCI image and registers the completed artifact.
|
||||
This is the blocking counterpart of :meth:`create_function_async`.
|
||||
LanceDB Embedded connections raise ``NotImplementedError``.
|
||||
@@ -1040,7 +1044,10 @@ class DBConnection(EnforceOverrides):
|
||||
```
|
||||
"""
|
||||
return self.create_function_async(
|
||||
definition, secrets=secrets, namespace_path=namespace_path
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
).wait()
|
||||
|
||||
def create_function_async(
|
||||
@@ -1048,10 +1055,14 @@ class DBConnection(EnforceOverrides):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
"""Submit a scalar Python function for building and registration.
|
||||
|
||||
``exist_ok=True`` preserves an existing named version and its metadata.
|
||||
The default retains the existing version-creation behavior.
|
||||
|
||||
The server-side job builds the OCI image, then registers the completed
|
||||
artifact. The returned job has type ``query`` and the SQL query UUID as
|
||||
its ID. Waiting on it returns the immutable Function version.
|
||||
@@ -2396,11 +2407,15 @@ class LanceDBConnection(DBConnection):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(
|
||||
self._conn.create_function_async(
|
||||
definition, secrets=secrets, namespace_path=namespace_path
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
@@ -3732,10 +3747,14 @@ class AsyncConnection(object):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> AsyncJob[FunctionVersion]:
|
||||
"""Submit a scalar Python function for building and registration.
|
||||
|
||||
``exist_ok=True`` preserves an existing named version and its metadata.
|
||||
The default retains the existing version-creation behavior.
|
||||
|
||||
The server-side job builds the OCI image, then registers the completed
|
||||
artifact. The returned job has type ``query`` and the SQL query UUID as
|
||||
its ID. Waiting on it returns the immutable Function version.
|
||||
@@ -3749,7 +3768,7 @@ class AsyncConnection(object):
|
||||
raise TypeError("create_function_async requires a @function definition")
|
||||
request = definition.bind_secrets(secrets)
|
||||
inner = await self._inner.create_function_async(
|
||||
request.to_canonical_json(), namespace_path
|
||||
request.to_canonical_json(), namespace_path, exist_ok=exist_ok
|
||||
)
|
||||
return _typed_job(inner, FunctionVersion.from_json)
|
||||
|
||||
|
||||
@@ -992,11 +992,15 @@ class RemoteDBConnection(DBConnection):
|
||||
definition: FunctionDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(
|
||||
self._conn.create_function_async(
|
||||
definition, secrets=secrets, namespace_path=namespace_path
|
||||
definition,
|
||||
secrets=secrets,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
from lancedb.functions import function
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("entry", ["sync", "sync_job", "async_job"])
|
||||
@pytest.mark.parametrize("exist_ok", [None, False, True])
|
||||
async def test_function_exist_ok_forwards_sql_and_keeps_stored_version(entry, exist_ok):
|
||||
flight = pytest.importorskip("pyarrow.flight")
|
||||
fixture = (
|
||||
Path(__file__).parents[3]
|
||||
/ "rust/lancedb/tests/fixtures/first_class_functions/v1"
|
||||
/ "remote_function_version.canonical.json"
|
||||
)
|
||||
stored = json.loads(fixture.read_text())
|
||||
result = pa.table({"definition": [json.dumps(stored)]})
|
||||
|
||||
@function(name=stored["name"])
|
||||
def replacement(value: float) -> float:
|
||||
return value * 2
|
||||
|
||||
class FunctionServer(flight.FlightServerBase):
|
||||
def __init__(self):
|
||||
super().__init__("grpc://127.0.0.1:0")
|
||||
self.requests = []
|
||||
|
||||
def do_get(self, context, ticket):
|
||||
self.requests.append(ticket.ticket)
|
||||
return flight.RecordBatchStream(result)
|
||||
|
||||
with FunctionServer() as server:
|
||||
options = {
|
||||
"api_key": "test-key",
|
||||
"host_override": "http://localhost:10024",
|
||||
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
|
||||
}
|
||||
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
|
||||
kwargs["namespace_path"] = ["team"]
|
||||
if entry == "async_job":
|
||||
db = await lancedb.connect_async("db://analytics", **options)
|
||||
job = await db.create_function_async(replacement, **kwargs)
|
||||
version = await job.wait()
|
||||
else:
|
||||
db = lancedb.connect("db://analytics", **options)
|
||||
if entry == "sync_job":
|
||||
version = db.create_function_async(replacement, **kwargs).wait()
|
||||
else:
|
||||
version = db.create_function(replacement, **kwargs)
|
||||
(request,) = server.requests
|
||||
|
||||
prefix = "CREATE FUNCTION IF NOT EXISTS" if exist_ok else "CREATE FUNCTION"
|
||||
assert f'{prefix} "{stored["name"]}" FROM '.encode() in request
|
||||
assert b'"exist_ok"' not in request
|
||||
assert json.loads(version.to_canonical_json()) == stored
|
||||
@@ -903,15 +903,17 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (request_json, namespace_path=None))]
|
||||
#[pyo3(signature = (request_json, namespace_path=None, *, exist_ok=false))]
|
||||
pub fn create_function_async(
|
||||
self_: PyRef<'_, Self>,
|
||||
request_json: String,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
|
||||
let mut request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json)
|
||||
.infer_error()?;
|
||||
request.exist_ok = exist_ok;
|
||||
let namespace_path = namespace_path.unwrap_or_default();
|
||||
future_into_py(self_.py(), async move {
|
||||
inner
|
||||
|
||||
@@ -834,6 +834,9 @@ fn validate_row_timeout_value(value: Option<f64>) -> std::result::Result<(), &'s
|
||||
/// database already holds, which Sophon resolves inside the remote runtime.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
pub struct FunctionRegistrationRequest {
|
||||
/// Preserve an existing named Function version instead of registering a new one.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub exist_ok: bool,
|
||||
/// Default retries of failed user-code calls. Enterprise only.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub retry_policy: Option<FunctionRetryPolicy>,
|
||||
@@ -876,6 +879,10 @@ pub struct FunctionRegistrationRequest {
|
||||
pub models: Vec<HuggingFaceModel>,
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
}
|
||||
|
||||
impl FunctionRegistrationRequest {
|
||||
/// Decode a registration request supplied by an SDK language binding.
|
||||
pub fn from_json(json: &str) -> Result<Self> {
|
||||
|
||||
@@ -34,13 +34,22 @@ impl SqlClient {
|
||||
validate_function_name(&request.name)?;
|
||||
validate_namespace(namespace)?;
|
||||
let name = quote_identifier(&request.name);
|
||||
let if_not_exists = if request.exist_ok {
|
||||
" IF NOT EXISTS"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let mut body = serde_json::to_value(request).map_err(|error| Error::Runtime {
|
||||
message: error.to_string(),
|
||||
})?;
|
||||
body.as_object_mut().unwrap().remove("name");
|
||||
body.as_object_mut().unwrap().remove("exist_ok");
|
||||
self.object_job(
|
||||
ExecuteQueryRequest {
|
||||
query: format!("CREATE FUNCTION {name} FROM {}", literal(&body.to_string())),
|
||||
query: format!(
|
||||
"CREATE FUNCTION{if_not_exists} {name} FROM {}",
|
||||
literal(&body.to_string())
|
||||
),
|
||||
default_namespace_path: namespace.to_vec(),
|
||||
parameters: None,
|
||||
},
|
||||
|
||||
@@ -3087,7 +3087,16 @@ async fn function_creation_and_description_use_sql_query_handles() {
|
||||
const REQUEST: &str = include_str!(
|
||||
"../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json"
|
||||
);
|
||||
for (defer, with_models) in [(false, false), (false, true), (true, false), (true, true)] {
|
||||
for (defer, with_models, exist_ok) in [
|
||||
(false, false, false),
|
||||
(false, true, false),
|
||||
(true, false, false),
|
||||
(true, true, false),
|
||||
(false, false, true),
|
||||
(false, true, true),
|
||||
(true, false, true),
|
||||
(true, true, true),
|
||||
] {
|
||||
let mut request_json: serde_json::Value = serde_json::from_str(REQUEST).unwrap();
|
||||
request_json["batch_size"] = serde_json::json!(2);
|
||||
request_json["max_concurrency"] = serde_json::json!(4);
|
||||
@@ -3113,9 +3122,11 @@ async fn function_creation_and_description_use_sql_query_handles() {
|
||||
let expected_models = version_json["image"].get("models");
|
||||
let mut expected_request = request_json.clone();
|
||||
expected_request.as_object_mut().unwrap().remove("name");
|
||||
let request =
|
||||
let mut request =
|
||||
crate::function::FunctionRegistrationRequest::from_json(&request_json.to_string())
|
||||
.unwrap();
|
||||
assert!(!request.exist_ok);
|
||||
request.exist_ok = exist_ok;
|
||||
let result = RecordBatch::try_from_iter(vec![(
|
||||
"definition",
|
||||
Arc::new(StringArray::from(vec![version_json.to_string()])) as ArrayRef,
|
||||
@@ -3151,7 +3162,10 @@ async fn function_creation_and_description_use_sql_query_handles() {
|
||||
{
|
||||
let seen = statements.lock().unwrap();
|
||||
let encoded = seen[0]
|
||||
.strip_prefix(r#"CREATE FUNCTION "normalize_score" FROM '"#)
|
||||
.strip_prefix(&format!(
|
||||
"CREATE FUNCTION{} \"normalize_score\" FROM '",
|
||||
if exist_ok { " IF NOT EXISTS" } else { "" }
|
||||
))
|
||||
.unwrap()
|
||||
.strip_suffix('\'')
|
||||
.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user