diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 77316dff2..46d6c9c1a 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -310,7 +310,7 @@ class FunctionVersion(_RemoteValue): runtime: PythonRuntimeSpec runtime_digest: str environment_digest: str - secret_bindings: Mapping[str, str] = {} + secret_env_bindings: Mapping[str, str] = {} created_at: str def __call__(self, **inputs: Any) -> FunctionApplication: @@ -375,7 +375,7 @@ class FunctionRegistrationRequest(_RemoteValue): """Stable remote registration envelope produced by :func:`udf`. Credential values deliberately have no field here. The only secret-shaped - thing a client sends is ``secret_bindings``: the name of a Secret the + thing a client sends is ``secret_env_bindings``: the name of a Secret the database already holds, which the remote service resolves at execution. """ @@ -383,7 +383,7 @@ class FunctionRegistrationRequest(_RemoteValue): artifact: FunctionArtifactRequest signature: FunctionSignature runtime: PythonRuntimeSpec - secret_bindings: Mapping[str, str] = {} + secret_env_bindings: Mapping[str, str] = {} class FunctionVersionRef(_OpenRemoteValue): @@ -534,12 +534,6 @@ class RefreshColumnResult(_RemoteValue): _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") _DECLARED_SECRET = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") -MAX_FUNCTION_SECRET_BINDINGS = 16 -"""A Function binds at most this many Secrets. - -Each bound Secret is one extra read on the launch path of every fragment, so -the count needs a bound for the same reason a credential needs a size limit. -""" _FUNCTION_BLOB_V2_TYPE = "blob_v2" _ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name" _BLOB_V2_EXTENSION_NAME = "lance.blob.v2" @@ -1310,11 +1304,6 @@ class UdfDefinition: f"Function secrets must be EnvVarSecret values, not {kinds!r}; a " "credential value is never sent to this API" ) - if len(bindings) > MAX_FUNCTION_SECRET_BINDINGS: - raise ValueError( - f"a Function binds at most {MAX_FUNCTION_SECRET_BINDINGS} secrets, " - f"not {len(bindings)}" - ) variables = [binding.env_variable for binding in bindings] duplicates = sorted({name for name in variables if variables.count(name) > 1}) if duplicates: @@ -1334,7 +1323,7 @@ class UdfDefinition: if not bindings: return self._request resolved = {binding.env_variable: binding.secret for binding in bindings} - return self._request._copy(update={"secret_bindings": resolved}) + return self._request._copy(update={"secret_env_bindings": resolved}) def __call__(self, *args, **kwargs): return self._function(*args, **kwargs) diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index ba1964e6c..b39884b70 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -110,7 +110,7 @@ def test_function_version_identity_is_immutable_and_exact(): version = FunctionVersion.from_json(json.dumps(value)) assert version.name == "embed" assert version.version == "fv_01K3EXACT" - assert dict(version.secret_bindings) == {"HF_TOKEN": "hf-prod"} + assert dict(version.secret_env_bindings) == {"HF_TOKEN": "hf-prod"} with pytest.raises((TypeError, ValueError)): version.version = "fv_changed" @@ -299,17 +299,17 @@ def test_canonical_client_values_carry_bindings_and_no_credentials(): json.dumps(job_result("remote_function_job.json")) ) canonical = json.loads(version.to_canonical_json()) - assert canonical["secret_bindings"] == {"HF_TOKEN": "hf-prod"} + assert canonical["secret_env_bindings"] == {"HF_TOKEN": "hf-prod"} assert_no_secret_values(canonical) def test_a_version_without_bindings_keeps_the_original_wire_shape(): """Every Function registered before Secrets existed serializes unchanged.""" value = job_result("remote_function_job.json") - del value["secret_bindings"] + del value["secret_env_bindings"] version = FunctionVersion.from_json(json.dumps(value)) - assert dict(version.secret_bindings) == {} - assert "secret_bindings" not in json.loads(version.to_canonical_json()) + assert dict(version.secret_env_bindings) == {} + assert "secret_env_bindings" not in json.loads(version.to_canonical_json()) class _FunctionDeclarationInner: diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 4499769a2..56598779e 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -21,6 +21,7 @@ import pyarrow as pa import pytest import lancedb +from lancedb.background_loop import LOOP from lancedb.functions import ( PythonRuntimeSpec, UdfDefinition, @@ -100,7 +101,7 @@ def test_an_unbound_request_carries_no_binding_at_all(): whether or not a credential is later bound to it. """ unbound = json.loads(analyze_caption.registration_request.to_canonical_json()) - assert "secret_bindings" not in unbound + assert "secret_env_bindings" not in unbound assert "OPENAI_API_KEY" not in json.dumps(unbound) @@ -120,7 +121,7 @@ def test_a_function_declaring_no_secret_is_registered_exactly_as_before(): == normalize_score.registration_request.to_canonical_json() ) assert ( - "secret_bindings" + "secret_env_bindings" not in normalize_score.registration_request.to_canonical_json() ) @@ -144,12 +145,37 @@ def test_bindings_may_not_collide_with_plain_configuration(): def test_a_function_binds_at_most_sixteen_secrets(): + """The cap lives in Rust, so no language surface can be talked past it. + + Registering through the typed API and hand-rolling the request envelope + reach the same boundary, and neither reaches the wire. + """ bindings = [ EnvVarSecret(secret=f"secret-{index}", env_variable=f"TOKEN_{index}") for index in range(17) ] - with pytest.raises(ValueError, match="at most 16 secrets"): - normalize_score.bind_secrets(bindings) + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + with pytest.raises(ValueError, match="at most 16 secrets"): + db.create_function(normalize_score, secrets=bindings) + + envelope = json.loads(normalize_score.registration_request.to_canonical_json()) + envelope["secret_env_bindings"] = { + f"TOKEN_{index}": f"secret-{index}" for index in range(17) + } + + async def submit_envelope(): + return await db._conn._inner.create_function_async(json.dumps(envelope)) + + with pytest.raises(ValueError, match="at most 16 secrets"): + LOOP.run(submit_envelope()) + + assert state["requests"] == [] def test_a_credential_value_is_rejected_in_the_binding_position(): @@ -1332,7 +1358,7 @@ def _mock_remote_function_catalog(): "runtime": body["runtime"], "runtime_digest": "sha256:runtime", "environment_digest": "sha256:environment", - "secret_bindings": body.get("secret_bindings", {}), + "secret_env_bindings": body.get("secret_env_bindings", {}), "created_at": "2026-08-21T00:00:00Z", } response = {"job_id": "job-register"} @@ -1443,10 +1469,10 @@ def test_remote_registration_sends_bindings_and_never_a_credential(): secrets=[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")], ) - assert dict(created.secret_bindings) == {"OPENAI_API_KEY": "openai-prod"} + assert dict(created.secret_env_bindings) == {"OPENAI_API_KEY": "openai-prod"} path, create_request = state["requests"][0] assert path == "/v1/functions/create" - assert create_request["secret_bindings"] == {"OPENAI_API_KEY": "openai-prod"} + assert create_request["secret_env_bindings"] == {"OPENAI_API_KEY": "openai-prod"} # The request names a Secret and carries nothing that could be one. assert create_request == json.loads( analyze_caption.bind_secrets( diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index b399d9081..1a10d0386 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -586,10 +586,15 @@ impl Connection { /// Registration is remote-only and always asynchronous. Waiting on the /// returned typed job yields the durable [`crate::function::FunctionVersion`]. /// Local databases return [`Error::NotSupported`]. + /// + /// The request's binding shape is validated here rather than in any one + /// language binding, so every client surface rejects the same envelopes + /// before one reaches the wire. pub async fn create_function_async( &self, request: crate::function::FunctionRegistrationRequest, ) -> Result> { + request.validate()?; self.internal.create_function_async(request).await } diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 1e75d4202..a9bc343ce 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -410,7 +410,7 @@ pub struct FunctionVersion { runtime_digest: String, environment_digest: String, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - secret_bindings: BTreeMap, + secret_env_bindings: BTreeMap, created_at: String, } @@ -449,8 +449,8 @@ impl FunctionVersion { /// them are not, and resolve at execution. Rotating a bound Secret /// therefore changes what the same version runs with, and no value has a /// field in this model. - pub fn secret_bindings(&self) -> &BTreeMap { - &self.secret_bindings + pub fn secret_env_bindings(&self) -> &BTreeMap { + &self.secret_env_bindings } pub fn created_at(&self) -> &str { @@ -493,10 +493,16 @@ pub struct FunctionArtifactRequest { pub adapter: PythonAdapterSpec, } +/// A Function binds at most this many Secrets to environment variables. +/// +/// Each bound Secret is one extra read on the launch path of every fragment, so +/// the count needs a bound for the same reason a credential needs a size limit. +pub const MAX_FUNCTION_SECRET_ENV_BINDINGS: usize = 16; + /// Stable request envelope for remote immutable Function registration. /// /// Credential values deliberately have no field here. The only secret-shaped -/// thing a client sends is `secret_bindings`: the name of a Secret the +/// thing a client sends is `secret_env_bindings`: the name of a Secret the /// database already holds, which Sophon resolves inside the remote runtime. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionRegistrationRequest { @@ -504,10 +510,30 @@ pub struct FunctionRegistrationRequest { pub artifact: FunctionArtifactRequest, pub signature: FunctionSignature, pub runtime: PythonRuntimeSpec, - /// Declared environment variable name to the Secret it binds. Every bound - /// Secret must already exist; registration fails otherwise. + /// Declared environment variable name to the Secret it binds. A binding is + /// a reference: whether the Secret exists is answered when a column is + /// declared against this version, not here. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub secret_bindings: BTreeMap, + pub secret_env_bindings: BTreeMap, +} + +impl FunctionRegistrationRequest { + /// Reject a registration whose bindings exceed what a launch can deliver. + /// + /// Shape only, and deliberately not a check that each bound Secret exists: + /// that is the service's answer, and it is asked for the first time when a + /// column is declared against the registered version. + pub fn validate(&self) -> Result<()> { + if self.secret_env_bindings.len() > MAX_FUNCTION_SECRET_ENV_BINDINGS { + return Err(Error::InvalidInput { + message: format!( + "a Function binds at most {MAX_FUNCTION_SECRET_ENV_BINDINGS} secrets, not {}", + self.secret_env_bindings.len() + ), + }); + } + Ok(()) + } } impl_json!(FunctionRegistrationRequest); diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index 92cb9e1ee..a3bdf73b8 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -50,7 +50,7 @@ fn function_version_job_result_matches_shared_canonical_golden() { assert_eq!(version.version(), "fv_01K3EXACT"); assert_eq!(version.runtime_digest(), "sha256:runtime"); assert_eq!( - version.secret_bindings(), + version.secret_env_bindings(), &BTreeMap::from([("HF_TOKEN".to_string(), "hf-prod".to_string())]) ); assert_eq!( @@ -180,7 +180,7 @@ fn canonical_client_values_carry_bindings_and_no_credentials() { .expect("canonical JSON"); assert_eq!( - canonical["secret_bindings"], + canonical["secret_env_bindings"], serde_json::json!({"HF_TOKEN": "hf-prod"}) ); assert_no_secret_values(&canonical); @@ -193,14 +193,14 @@ fn a_version_without_bindings_keeps_the_original_wire_shape() { result .as_object_mut() .expect("Function version object") - .remove("secret_bindings"); + .remove("secret_env_bindings"); let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); - assert!(version.secret_bindings().is_empty()); + assert!(version.secret_env_bindings().is_empty()); assert!( !version .to_canonical_json() .expect("canonical FunctionVersion") - .contains("secret_bindings") + .contains("secret_env_bindings") ); } diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs index c1fbb559c..500028d75 100644 --- a/rust/lancedb/tests/first_class_function_slice2.rs +++ b/rust/lancedb/tests/first_class_function_slice2.rs @@ -5,7 +5,7 @@ use std::fs; use std::path::PathBuf; use lancedb::Error; -use lancedb::function::FunctionRegistrationRequest; +use lancedb::function::{FunctionRegistrationRequest, MAX_FUNCTION_SECRET_ENV_BINDINGS}; use serde_json::Value; fn fixture(name: &str) -> String { @@ -44,8 +44,8 @@ fn registration_request_matches_shared_canonical_golden() { assert_eq!(request.name, "normalize_score"); assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch"); // The unchanged path: a Function that binds nothing serializes today's - // bytes, with no `secret_bindings` key at all. - assert!(request.secret_bindings.is_empty()); + // bytes, with no `secret_env_bindings` key at all. + assert!(request.secret_env_bindings.is_empty()); assert_eq!( request.to_canonical_json().expect("canonical request"), fixture("remote_function_registration_request.canonical.json").trim() @@ -67,7 +67,7 @@ fn secret_bound_registration_request_matches_shared_canonical_golden() { .expect("registration request"); assert_eq!(request.name, "analyze_caption"); assert_eq!( - request.secret_bindings, + request.secret_env_bindings, std::collections::BTreeMap::from([( "OPENAI_API_KEY".to_string(), "openai-prod".to_string() @@ -113,3 +113,28 @@ async fn local_function_catalog_operations_return_stable_not_supported() { )); } } + +/// The cap is enforced above the backend, so every database and every language +/// surface rejects the same envelope. A local connection would otherwise answer +/// `NotSupported` first, which is what makes it the honest probe here. +#[tokio::test] +async fn a_function_binds_at_most_sixteen_secrets() { + let directory = tempfile::tempdir().unwrap(); + let connection = lancedb::connect(directory.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let mut request = FunctionRegistrationRequest::from_json(&fixture( + "remote_function_registration_request.json", + )) + .unwrap(); + request.secret_env_bindings = (0..=MAX_FUNCTION_SECRET_ENV_BINDINGS) + .map(|index| (format!("TOKEN_{index}"), format!("secret-{index}"))) + .collect(); + + let error = connection.create_function_async(request).await.unwrap_err(); + assert!(matches!( + error, + Error::InvalidInput { message } if message.contains("at most 16 secrets") + )); +} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json index b5103ba05..229511cee 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json @@ -24,7 +24,7 @@ }, "runtime_digest": "sha256:runtime", "environment_digest": "sha256:environment", - "secret_bindings": {"HF_TOKEN": "hf-prod"}, + "secret_env_bindings": {"HF_TOKEN": "hf-prod"}, "created_at": "2026-08-21T00:00:00Z" }, "future_job": {"trace_id": "trace-1"} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json index 7c971de00..3a879bccd 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json @@ -1 +1 @@ -{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_bindings":{"OPENAI_API_KEY":"openai-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}} +{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_env_bindings":{"OPENAI_API_KEY":"openai-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json index 679b831a6..05ac5f348 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json @@ -26,7 +26,7 @@ "kind": "python", "python_version": "3.12" }, - "secret_bindings": { + "secret_env_bindings": { "OPENAI_API_KEY": "openai-prod" }, "signature": { diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json index e182bc2af..ef7868a33 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json @@ -1 +1 @@ -{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","secret_bindings":{"HF_TOKEN":"hf-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"} +{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","secret_env_bindings":{"HF_TOKEN":"hf-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}