refactor(secrets): name the binding field for its delivery mode, cap in Rust

`secret_bindings` read as the general record of what a Function needs, but a
`Map<String, String>` keyed on environment-variable name can only ever hold an
env-delivered binding: an accessor binding has no variable to key on. Naming
the field for its delivery mode leaves a later mode a sibling field rather than
a tagged value type, which would break a field already in the identity hash.

The per-Function cap moves from `bind_secrets` to
`Connection::create_function_async`, above the backend dispatch, so it holds
for every language surface rather than only the one that validates first. The
low-level PyO3 entry point reached the wire past the Python check; it no longer
does.

Also corrects a doc comment claiming registration fails when a bound Secret is
absent. Existence is first answered at `add_columns`, by design.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UfmeJ533rQDnPBkMtjerV6
This commit is contained in:
Jonathan M Hsieh
2026-09-09 16:59:40 +00:00
co-authored by Claude Opus 5
parent 7b29fb2f51
commit 4d09f8ce26
11 changed files with 118 additions and 47 deletions
+4 -15
View File
@@ -310,7 +310,7 @@ class FunctionVersion(_RemoteValue):
runtime: PythonRuntimeSpec
runtime_digest: str
environment_digest: str
secret_bindings: Mapping[str, str] = {}
secret_env_bindings: Mapping[str, str] = {}
created_at: str
def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -375,7 +375,7 @@ class FunctionRegistrationRequest(_RemoteValue):
"""Stable remote registration envelope produced by :func:`udf`.
Credential values deliberately have no field here. The only secret-shaped
thing a client sends is ``secret_bindings``: the name of a Secret the
thing a client sends is ``secret_env_bindings``: the name of a Secret the
database already holds, which the remote service resolves at execution.
"""
@@ -383,7 +383,7 @@ class FunctionRegistrationRequest(_RemoteValue):
artifact: FunctionArtifactRequest
signature: FunctionSignature
runtime: PythonRuntimeSpec
secret_bindings: Mapping[str, str] = {}
secret_env_bindings: Mapping[str, str] = {}
class FunctionVersionRef(_OpenRemoteValue):
@@ -534,12 +534,6 @@ class RefreshColumnResult(_RemoteValue):
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
_DECLARED_SECRET = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
MAX_FUNCTION_SECRET_BINDINGS = 16
"""A Function binds at most this many Secrets.
Each bound Secret is one extra read on the launch path of every fragment, so
the count needs a bound for the same reason a credential needs a size limit.
"""
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
@@ -1310,11 +1304,6 @@ class UdfDefinition:
f"Function secrets must be EnvVarSecret values, not {kinds!r}; a "
"credential value is never sent to this API"
)
if len(bindings) > MAX_FUNCTION_SECRET_BINDINGS:
raise ValueError(
f"a Function binds at most {MAX_FUNCTION_SECRET_BINDINGS} secrets, "
f"not {len(bindings)}"
)
variables = [binding.env_variable for binding in bindings]
duplicates = sorted({name for name in variables if variables.count(name) > 1})
if duplicates:
@@ -1334,7 +1323,7 @@ class UdfDefinition:
if not bindings:
return self._request
resolved = {binding.env_variable: binding.secret for binding in bindings}
return self._request._copy(update={"secret_bindings": resolved})
return self._request._copy(update={"secret_env_bindings": resolved})
def __call__(self, *args, **kwargs):
return self._function(*args, **kwargs)
@@ -110,7 +110,7 @@ def test_function_version_identity_is_immutable_and_exact():
version = FunctionVersion.from_json(json.dumps(value))
assert version.name == "embed"
assert version.version == "fv_01K3EXACT"
assert dict(version.secret_bindings) == {"HF_TOKEN": "hf-prod"}
assert dict(version.secret_env_bindings) == {"HF_TOKEN": "hf-prod"}
with pytest.raises((TypeError, ValueError)):
version.version = "fv_changed"
@@ -299,17 +299,17 @@ def test_canonical_client_values_carry_bindings_and_no_credentials():
json.dumps(job_result("remote_function_job.json"))
)
canonical = json.loads(version.to_canonical_json())
assert canonical["secret_bindings"] == {"HF_TOKEN": "hf-prod"}
assert canonical["secret_env_bindings"] == {"HF_TOKEN": "hf-prod"}
assert_no_secret_values(canonical)
def test_a_version_without_bindings_keeps_the_original_wire_shape():
"""Every Function registered before Secrets existed serializes unchanged."""
value = job_result("remote_function_job.json")
del value["secret_bindings"]
del value["secret_env_bindings"]
version = FunctionVersion.from_json(json.dumps(value))
assert dict(version.secret_bindings) == {}
assert "secret_bindings" not in json.loads(version.to_canonical_json())
assert dict(version.secret_env_bindings) == {}
assert "secret_env_bindings" not in json.loads(version.to_canonical_json())
class _FunctionDeclarationInner:
@@ -21,6 +21,7 @@ import pyarrow as pa
import pytest
import lancedb
from lancedb.background_loop import LOOP
from lancedb.functions import (
PythonRuntimeSpec,
UdfDefinition,
@@ -100,7 +101,7 @@ def test_an_unbound_request_carries_no_binding_at_all():
whether or not a credential is later bound to it.
"""
unbound = json.loads(analyze_caption.registration_request.to_canonical_json())
assert "secret_bindings" not in unbound
assert "secret_env_bindings" not in unbound
assert "OPENAI_API_KEY" not in json.dumps(unbound)
@@ -120,7 +121,7 @@ def test_a_function_declaring_no_secret_is_registered_exactly_as_before():
== normalize_score.registration_request.to_canonical_json()
)
assert (
"secret_bindings"
"secret_env_bindings"
not in normalize_score.registration_request.to_canonical_json()
)
@@ -144,12 +145,37 @@ def test_bindings_may_not_collide_with_plain_configuration():
def test_a_function_binds_at_most_sixteen_secrets():
"""The cap lives in Rust, so no language surface can be talked past it.
Registering through the typed API and hand-rolling the request envelope
reach the same boundary, and neither reaches the wire.
"""
bindings = [
EnvVarSecret(secret=f"secret-{index}", env_variable=f"TOKEN_{index}")
for index in range(17)
]
with pytest.raises(ValueError, match="at most 16 secrets"):
normalize_score.bind_secrets(bindings)
with _mock_remote_function_catalog() as (host, state):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
with pytest.raises(ValueError, match="at most 16 secrets"):
db.create_function(normalize_score, secrets=bindings)
envelope = json.loads(normalize_score.registration_request.to_canonical_json())
envelope["secret_env_bindings"] = {
f"TOKEN_{index}": f"secret-{index}" for index in range(17)
}
async def submit_envelope():
return await db._conn._inner.create_function_async(json.dumps(envelope))
with pytest.raises(ValueError, match="at most 16 secrets"):
LOOP.run(submit_envelope())
assert state["requests"] == []
def test_a_credential_value_is_rejected_in_the_binding_position():
@@ -1332,7 +1358,7 @@ def _mock_remote_function_catalog():
"runtime": body["runtime"],
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"secret_bindings": body.get("secret_bindings", {}),
"secret_env_bindings": body.get("secret_env_bindings", {}),
"created_at": "2026-08-21T00:00:00Z",
}
response = {"job_id": "job-register"}
@@ -1443,10 +1469,10 @@ def test_remote_registration_sends_bindings_and_never_a_credential():
secrets=[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")],
)
assert dict(created.secret_bindings) == {"OPENAI_API_KEY": "openai-prod"}
assert dict(created.secret_env_bindings) == {"OPENAI_API_KEY": "openai-prod"}
path, create_request = state["requests"][0]
assert path == "/v1/functions/create"
assert create_request["secret_bindings"] == {"OPENAI_API_KEY": "openai-prod"}
assert create_request["secret_env_bindings"] == {"OPENAI_API_KEY": "openai-prod"}
# The request names a Secret and carries nothing that could be one.
assert create_request == json.loads(
analyze_caption.bind_secrets(
+5
View File
@@ -586,10 +586,15 @@ impl Connection {
/// Registration is remote-only and always asynchronous. Waiting on the
/// returned typed job yields the durable [`crate::function::FunctionVersion`].
/// Local databases return [`Error::NotSupported`].
///
/// The request's binding shape is validated here rather than in any one
/// language binding, so every client surface rejects the same envelopes
/// before one reaches the wire.
pub async fn create_function_async(
&self,
request: crate::function::FunctionRegistrationRequest,
) -> Result<crate::job::Job<crate::function::FunctionVersion>> {
request.validate()?;
self.internal.create_function_async(request).await
}
+33 -7
View File
@@ -410,7 +410,7 @@ pub struct FunctionVersion {
runtime_digest: String,
environment_digest: String,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
secret_bindings: BTreeMap<String, String>,
secret_env_bindings: BTreeMap<String, String>,
created_at: String,
}
@@ -449,8 +449,8 @@ impl FunctionVersion {
/// them are not, and resolve at execution. Rotating a bound Secret
/// therefore changes what the same version runs with, and no value has a
/// field in this model.
pub fn secret_bindings(&self) -> &BTreeMap<String, String> {
&self.secret_bindings
pub fn secret_env_bindings(&self) -> &BTreeMap<String, String> {
&self.secret_env_bindings
}
pub fn created_at(&self) -> &str {
@@ -493,10 +493,16 @@ pub struct FunctionArtifactRequest {
pub adapter: PythonAdapterSpec,
}
/// A Function binds at most this many Secrets to environment variables.
///
/// Each bound Secret is one extra read on the launch path of every fragment, so
/// the count needs a bound for the same reason a credential needs a size limit.
pub const MAX_FUNCTION_SECRET_ENV_BINDINGS: usize = 16;
/// Stable request envelope for remote immutable Function registration.
///
/// Credential values deliberately have no field here. The only secret-shaped
/// thing a client sends is `secret_bindings`: the name of a Secret the
/// thing a client sends is `secret_env_bindings`: the name of a Secret the
/// database already holds, which Sophon resolves inside the remote runtime.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionRegistrationRequest {
@@ -504,10 +510,30 @@ pub struct FunctionRegistrationRequest {
pub artifact: FunctionArtifactRequest,
pub signature: FunctionSignature,
pub runtime: PythonRuntimeSpec,
/// Declared environment variable name to the Secret it binds. Every bound
/// Secret must already exist; registration fails otherwise.
/// Declared environment variable name to the Secret it binds. A binding is
/// a reference: whether the Secret exists is answered when a column is
/// declared against this version, not here.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub secret_bindings: BTreeMap<String, String>,
pub secret_env_bindings: BTreeMap<String, String>,
}
impl FunctionRegistrationRequest {
/// Reject a registration whose bindings exceed what a launch can deliver.
///
/// Shape only, and deliberately not a check that each bound Secret exists:
/// that is the service's answer, and it is asked for the first time when a
/// column is declared against the registered version.
pub fn validate(&self) -> Result<()> {
if self.secret_env_bindings.len() > MAX_FUNCTION_SECRET_ENV_BINDINGS {
return Err(Error::InvalidInput {
message: format!(
"a Function binds at most {MAX_FUNCTION_SECRET_ENV_BINDINGS} secrets, not {}",
self.secret_env_bindings.len()
),
});
}
Ok(())
}
}
impl_json!(FunctionRegistrationRequest);
@@ -50,7 +50,7 @@ fn function_version_job_result_matches_shared_canonical_golden() {
assert_eq!(version.version(), "fv_01K3EXACT");
assert_eq!(version.runtime_digest(), "sha256:runtime");
assert_eq!(
version.secret_bindings(),
version.secret_env_bindings(),
&BTreeMap::from([("HF_TOKEN".to_string(), "hf-prod".to_string())])
);
assert_eq!(
@@ -180,7 +180,7 @@ fn canonical_client_values_carry_bindings_and_no_credentials() {
.expect("canonical JSON");
assert_eq!(
canonical["secret_bindings"],
canonical["secret_env_bindings"],
serde_json::json!({"HF_TOKEN": "hf-prod"})
);
assert_no_secret_values(&canonical);
@@ -193,14 +193,14 @@ fn a_version_without_bindings_keeps_the_original_wire_shape() {
result
.as_object_mut()
.expect("Function version object")
.remove("secret_bindings");
.remove("secret_env_bindings");
let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result");
assert!(version.secret_bindings().is_empty());
assert!(version.secret_env_bindings().is_empty());
assert!(
!version
.to_canonical_json()
.expect("canonical FunctionVersion")
.contains("secret_bindings")
.contains("secret_env_bindings")
);
}
@@ -5,7 +5,7 @@ use std::fs;
use std::path::PathBuf;
use lancedb::Error;
use lancedb::function::FunctionRegistrationRequest;
use lancedb::function::{FunctionRegistrationRequest, MAX_FUNCTION_SECRET_ENV_BINDINGS};
use serde_json::Value;
fn fixture(name: &str) -> String {
@@ -44,8 +44,8 @@ fn registration_request_matches_shared_canonical_golden() {
assert_eq!(request.name, "normalize_score");
assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch");
// The unchanged path: a Function that binds nothing serializes today's
// bytes, with no `secret_bindings` key at all.
assert!(request.secret_bindings.is_empty());
// bytes, with no `secret_env_bindings` key at all.
assert!(request.secret_env_bindings.is_empty());
assert_eq!(
request.to_canonical_json().expect("canonical request"),
fixture("remote_function_registration_request.canonical.json").trim()
@@ -67,7 +67,7 @@ fn secret_bound_registration_request_matches_shared_canonical_golden() {
.expect("registration request");
assert_eq!(request.name, "analyze_caption");
assert_eq!(
request.secret_bindings,
request.secret_env_bindings,
std::collections::BTreeMap::from([(
"OPENAI_API_KEY".to_string(),
"openai-prod".to_string()
@@ -113,3 +113,28 @@ async fn local_function_catalog_operations_return_stable_not_supported() {
));
}
}
/// The cap is enforced above the backend, so every database and every language
/// surface rejects the same envelope. A local connection would otherwise answer
/// `NotSupported` first, which is what makes it the honest probe here.
#[tokio::test]
async fn a_function_binds_at_most_sixteen_secrets() {
let directory = tempfile::tempdir().unwrap();
let connection = lancedb::connect(directory.path().to_str().unwrap())
.execute()
.await
.unwrap();
let mut request = FunctionRegistrationRequest::from_json(&fixture(
"remote_function_registration_request.json",
))
.unwrap();
request.secret_env_bindings = (0..=MAX_FUNCTION_SECRET_ENV_BINDINGS)
.map(|index| (format!("TOKEN_{index}"), format!("secret-{index}")))
.collect();
let error = connection.create_function_async(request).await.unwrap_err();
assert!(matches!(
error,
Error::InvalidInput { message } if message.contains("at most 16 secrets")
));
}
@@ -24,7 +24,7 @@
},
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"secret_bindings": {"HF_TOKEN": "hf-prod"},
"secret_env_bindings": {"HF_TOKEN": "hf-prod"},
"created_at": "2026-08-21T00:00:00Z"
},
"future_job": {"trace_id": "trace-1"}
@@ -1 +1 @@
{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_bindings":{"OPENAI_API_KEY":"openai-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}}
{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_env_bindings":{"OPENAI_API_KEY":"openai-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}}
@@ -26,7 +26,7 @@
"kind": "python",
"python_version": "3.12"
},
"secret_bindings": {
"secret_env_bindings": {
"OPENAI_API_KEY": "openai-prod"
},
"signature": {
@@ -1 +1 @@
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","secret_bindings":{"HF_TOKEN":"hf-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","secret_env_bindings":{"HF_TOKEN":"hf-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}