diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 04c146bb1..aae67ceb9 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -111,6 +111,19 @@ with `db.get_function(name, version=...)`. The version may be an integer or its string form. Omit `version` to resolve and return the latest currently visible revision. +{/* Draft for the function exist_ok release. + +Use `exist_ok=True` on `create_function` or `create_function_async` to return +the existing named version without changing its code, credentials or settings. +This requires a deployment that supports the option. The default retains the +current behavior of registering a new version. + +```python +version = db.create_function(embed, exist_ok=True) +``` + +*/} + The SQL command submits an internal `create_function` image-build job and waits for publication. The SDK returns the SQL query handle. Query handles diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index 95c2d4d6e..55e2d6fe7 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -16,6 +16,15 @@ availability: ## CREATE FUNCTION {#create} +{/* Draft for the function exist_ok release. + +`CREATE FUNCTION IF NOT EXISTS` preserves an existing named Function version, +including its code, credentials and execution settings. In Python, pass +`exist_ok=True` to `create_function` or `create_function_async`. The default +retains the existing version-creation behavior. + +*/} + ```sql SQL icon="database" CREATE FUNCTION f(x int64) RETURNS int64 diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 56b644abb..44523fd42 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -188,7 +188,11 @@ class Connection(object): async def server_info(self) -> str: ... async def list_gpu_classes(self) -> list[GpuClass]: ... async def create_function_async( - self, request_json: str, namespace_path: Optional[List[str]] = None + self, + request_json: str, + namespace_path: Optional[List[str]] = None, + *, + exist_ok: bool = False, ) -> Job: ... async def get_function( self, diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 020c79be0..050b20727 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -1004,10 +1004,14 @@ class DBConnection(EnforceOverrides): definition: FunctionDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> FunctionVersion: """Build and register a scalar Python function, then return its version. + ``exist_ok=True`` returns the existing named Function version unchanged. + By default, registration retains its existing version-creation behavior. + The server builds the OCI image and registers the completed artifact. This is the blocking counterpart of :meth:`create_function_async`. LanceDB Embedded connections raise ``NotImplementedError``. @@ -1040,7 +1044,10 @@ class DBConnection(EnforceOverrides): ``` """ return self.create_function_async( - definition, secrets=secrets, namespace_path=namespace_path + definition, + secrets=secrets, + exist_ok=exist_ok, + namespace_path=namespace_path, ).wait() def create_function_async( @@ -1048,10 +1055,14 @@ class DBConnection(EnforceOverrides): definition: FunctionDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> Job[FunctionVersion]: """Submit a scalar Python function for building and registration. + ``exist_ok=True`` preserves an existing named version and its metadata. + The default retains the existing version-creation behavior. + The server-side job builds the OCI image, then registers the completed artifact. The returned job has type ``query`` and the SQL query UUID as its ID. Waiting on it returns the immutable Function version. @@ -2396,11 +2407,15 @@ class LanceDBConnection(DBConnection): definition: FunctionDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> Job[FunctionVersion]: job = LOOP.run( self._conn.create_function_async( - definition, secrets=secrets, namespace_path=namespace_path + definition, + secrets=secrets, + exist_ok=exist_ok, + namespace_path=namespace_path, ) ) return Job(job) @@ -3732,10 +3747,14 @@ class AsyncConnection(object): definition: FunctionDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> AsyncJob[FunctionVersion]: """Submit a scalar Python function for building and registration. + ``exist_ok=True`` preserves an existing named version and its metadata. + The default retains the existing version-creation behavior. + The server-side job builds the OCI image, then registers the completed artifact. The returned job has type ``query`` and the SQL query UUID as its ID. Waiting on it returns the immutable Function version. @@ -3749,7 +3768,7 @@ class AsyncConnection(object): raise TypeError("create_function_async requires a @function definition") request = definition.bind_secrets(secrets) inner = await self._inner.create_function_async( - request.to_canonical_json(), namespace_path + request.to_canonical_json(), namespace_path, exist_ok=exist_ok ) return _typed_job(inner, FunctionVersion.from_json) diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index ea9468331..870b85b1e 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -992,11 +992,15 @@ class RemoteDBConnection(DBConnection): definition: FunctionDefinition, *, secrets: Optional[Sequence[EnvVarSecret]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> Job[FunctionVersion]: job = LOOP.run( self._conn.create_function_async( - definition, secrets=secrets, namespace_path=namespace_path + definition, + secrets=secrets, + exist_ok=exist_ok, + namespace_path=namespace_path, ) ) return Job(job) diff --git a/python/python/tests/test_function_creation.py b/python/python/tests/test_function_creation.py new file mode 100644 index 000000000..823dba7b9 --- /dev/null +++ b/python/python/tests/test_function_creation.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import json +from pathlib import Path + +import pyarrow as pa +import pytest + +import lancedb +from lancedb.functions import function + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entry", ["sync", "sync_job", "async_job"]) +@pytest.mark.parametrize("exist_ok", [None, False, True]) +async def test_function_exist_ok_forwards_sql_and_keeps_stored_version(entry, exist_ok): + flight = pytest.importorskip("pyarrow.flight") + fixture = ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1" + / "remote_function_version.canonical.json" + ) + stored = json.loads(fixture.read_text()) + result = pa.table({"definition": [json.dumps(stored)]}) + + @function(name=stored["name"]) + def replacement(value: float) -> float: + return value * 2 + + class FunctionServer(flight.FlightServerBase): + def __init__(self): + super().__init__("grpc://127.0.0.1:0") + self.requests = [] + + def do_get(self, context, ticket): + self.requests.append(ticket.ticket) + return flight.RecordBatchStream(result) + + with FunctionServer() as server: + options = { + "api_key": "test-key", + "host_override": "http://localhost:10024", + "sql_host_override": f"grpc://127.0.0.1:{server.port}", + } + kwargs = {} if exist_ok is None else {"exist_ok": exist_ok} + kwargs["namespace_path"] = ["team"] + if entry == "async_job": + db = await lancedb.connect_async("db://analytics", **options) + job = await db.create_function_async(replacement, **kwargs) + version = await job.wait() + else: + db = lancedb.connect("db://analytics", **options) + if entry == "sync_job": + version = db.create_function_async(replacement, **kwargs).wait() + else: + version = db.create_function(replacement, **kwargs) + (request,) = server.requests + + prefix = "CREATE FUNCTION IF NOT EXISTS" if exist_ok else "CREATE FUNCTION" + assert f'{prefix} "{stored["name"]}" FROM '.encode() in request + assert b'"exist_ok"' not in request + assert json.loads(version.to_canonical_json()) == stored diff --git a/python/src/connection.rs b/python/src/connection.rs index ffa59fd02..dabecc048 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -903,15 +903,17 @@ impl Connection { }) } - #[pyo3(signature = (request_json, namespace_path=None))] + #[pyo3(signature = (request_json, namespace_path=None, *, exist_ok=false))] pub fn create_function_async( self_: PyRef<'_, Self>, request_json: String, namespace_path: Option>, + exist_ok: bool, ) -> PyResult> { let inner = self_.get_inner()?.clone(); - let request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json) + let mut request = lancedb::function::FunctionRegistrationRequest::from_json(&request_json) .infer_error()?; + request.exist_ok = exist_ok; let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { inner diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index e3d47948e..d1b63f2ad 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -834,6 +834,9 @@ fn validate_row_timeout_value(value: Option) -> std::result::Result<(), &'s /// database already holds, which Sophon resolves inside the remote runtime. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct FunctionRegistrationRequest { + /// Preserve an existing named Function version instead of registering a new one. + #[serde(default, skip_serializing_if = "is_false")] + pub exist_ok: bool, /// Default retries of failed user-code calls. Enterprise only. #[serde(default, skip_serializing_if = "Option::is_none")] pub retry_policy: Option, @@ -876,6 +879,10 @@ pub struct FunctionRegistrationRequest { pub models: Vec, } +fn is_false(value: &bool) -> bool { + !*value +} + impl FunctionRegistrationRequest { /// Decode a registration request supplied by an SDK language binding. pub fn from_json(json: &str) -> Result { diff --git a/rust/lancedb/src/remote/sql/function.rs b/rust/lancedb/src/remote/sql/function.rs index c2b5aba06..19a1a6cbc 100644 --- a/rust/lancedb/src/remote/sql/function.rs +++ b/rust/lancedb/src/remote/sql/function.rs @@ -34,13 +34,22 @@ impl SqlClient { validate_function_name(&request.name)?; validate_namespace(namespace)?; let name = quote_identifier(&request.name); + let if_not_exists = if request.exist_ok { + " IF NOT EXISTS" + } else { + "" + }; let mut body = serde_json::to_value(request).map_err(|error| Error::Runtime { message: error.to_string(), })?; body.as_object_mut().unwrap().remove("name"); + body.as_object_mut().unwrap().remove("exist_ok"); self.object_job( ExecuteQueryRequest { - query: format!("CREATE FUNCTION {name} FROM {}", literal(&body.to_string())), + query: format!( + "CREATE FUNCTION{if_not_exists} {name} FROM {}", + literal(&body.to_string()) + ), default_namespace_path: namespace.to_vec(), parameters: None, }, diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index 6a3f30390..e1780f118 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -3087,7 +3087,16 @@ async fn function_creation_and_description_use_sql_query_handles() { const REQUEST: &str = include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_registration_request.json" ); - for (defer, with_models) in [(false, false), (false, true), (true, false), (true, true)] { + for (defer, with_models, exist_ok) in [ + (false, false, false), + (false, true, false), + (true, false, false), + (true, true, false), + (false, false, true), + (false, true, true), + (true, false, true), + (true, true, true), + ] { let mut request_json: serde_json::Value = serde_json::from_str(REQUEST).unwrap(); request_json["batch_size"] = serde_json::json!(2); request_json["max_concurrency"] = serde_json::json!(4); @@ -3113,9 +3122,11 @@ async fn function_creation_and_description_use_sql_query_handles() { let expected_models = version_json["image"].get("models"); let mut expected_request = request_json.clone(); expected_request.as_object_mut().unwrap().remove("name"); - let request = + let mut request = crate::function::FunctionRegistrationRequest::from_json(&request_json.to_string()) .unwrap(); + assert!(!request.exist_ok); + request.exist_ok = exist_ok; let result = RecordBatch::try_from_iter(vec![( "definition", Arc::new(StringArray::from(vec![version_json.to_string()])) as ArrayRef, @@ -3151,7 +3162,10 @@ async fn function_creation_and_description_use_sql_query_handles() { { let seen = statements.lock().unwrap(); let encoded = seen[0] - .strip_prefix(r#"CREATE FUNCTION "normalize_score" FROM '"#) + .strip_prefix(&format!( + "CREATE FUNCTION{} \"normalize_score\" FROM '", + if exist_ok { " IF NOT EXISTS" } else { "" } + )) .unwrap() .strip_suffix('\'') .unwrap();