rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-11 15:52:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(secrets): address Secrets by namespace path

Browse Source 
A binding names a Secret by its parts -- `{name, namespace_path}` -- rather
than by one string that has to be parsed back apart. A joined id needs a
delimiter, and a delimiter has to be excluded from every name and every segment
forever, agreed on by both sides, and re-agreed each time either grows a new way
to be configured. `ClientConfig.id_delimiter` is exactly that: it already
governs table identifiers and is advertised on every request, so an id joined
on a fixed `$` would contradict a delimiter the same request declares.

Naming the parts settles all of it at the cost of one object. Nothing is
parsed, so nothing can parse two ways; the charset is the service's storage
rule rather than a delimiter's shadow; and the Secret verbs already send
`namespace_path` as a field, so the binding now matches them instead of being
the one place that flattens.

A root binding omits the path entirely, so its wire shape is byte-identical to
one written before namespaces existed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UfmeJ533rQDnPBkMtjerV6
This commit is contained in:
Jonathan M Hsieh
2026-09-11 15:38:12 +00:00
co-authored by Claude Opus 5
parent e9586850b6
commit f75cd1e412
18 changed files with 541 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/_lancedb.pyi
+11 -5
View File
@@ -153,11 +153,17 @@ class Connection(object):
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def create_secret(self, name: str, value: str) -> None: ...
async def alter_secret(self, name: str, value: str) -> None: ...
async def list_secrets(self) -> List[str]: ...
async def drop_secret(self, name: str) -> None: ...
async def describe_secret(self, name: str) -> Dict[str, str]: ...
async def create_secret(
self, name: str, value: str, namespace_path: List[str]
) -> None: ...
async def alter_secret(
self, name: str, value: str, namespace_path: List[str]
) -> None: ...
async def list_secrets(self, namespace_path: List[str]) -> List[str]: ...
async def drop_secret(self, name: str, namespace_path: List[str]) -> None: ...
async def describe_secret(
self, name: str, namespace_path: List[str]
) -> Dict[str, str]: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def execute_query_async(
python/python/lancedb/db.py
+72 -26
View File
@@ -58,7 +58,12 @@ from .materialized_view import (
SelectArg,
normalize_select,
)
from .secrets import EnvVarSecret, SecretInfo, validate_secret_name
from .secrets import (
EnvVarSecret,
SecretInfo,
validate_namespace_path,
validate_secret_name,
)
from .table import (
AsyncTable,
LanceTable,
@@ -779,7 +784,9 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def create_secret(self, name: str, value: str) -> None:
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Create a named Secret in this database.
Fails if the name is taken, so a create never silently becomes a
@@ -791,7 +798,9 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def alter_secret(self, name: str, value: str) -> None:
def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Replace the credential behind an existing Secret.
Fails if it does not exist. Every Function bound to the Secret uses the
@@ -803,7 +812,7 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def list_secrets(self) -> List[str]:
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
"""The names of every Secret in this database.
Names only. No method returns a stored credential, by construction
@@ -813,7 +822,9 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def drop_secret(self, name: str) -> None:
def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Drop a Secret.
Functions bound to it fail at their next job, naming the Secret; that
@@ -825,7 +836,9 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def describe_secret(self, name: str) -> SecretInfo:
def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
"""What this database records about a Secret: name and timestamps.
Never the value -- there is no code path that could return one. Local
@@ -1569,24 +1582,32 @@ class LanceDBConnection(DBConnection):
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def create_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.create_secret(name, value))
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path))
@override
def alter_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.alter_secret(name, value))
def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
@override
def list_secrets(self) -> List[str]:
return LOOP.run(self._conn.list_secrets())
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
@override
def drop_secret(self, name: str) -> None:
LOOP.run(self._conn.drop_secret(name))
def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
@override
def describe_secret(self, name: str) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name))
def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
@override
def list_jobs(self) -> List[JobInfo]:
@@ -2418,34 +2439,59 @@ class AsyncConnection(object):
"""Drop one exact immutable Function version from the remote catalog."""
return await self._inner.drop_function(name, version)
async def create_secret(self, name: str, value: str) -> None:
async def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Create a named Secret in this database.
Fails if the name is taken, so a create never silently becomes a
rotation. Nothing reads the value back.
"""
await self._inner.create_secret(validate_secret_name(name), value)
await self._inner.create_secret(
validate_secret_name(name),
value,
list(validate_namespace_path(namespace_path)),
)
async def alter_secret(self, name: str, value: str) -> None:
async def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Replace the credential behind an existing Secret.
Fails if it does not exist. Bound Functions use the new value from
their next job, with no new Function version.
"""
await self._inner.alter_secret(validate_secret_name(name), value)
await self._inner.alter_secret(
validate_secret_name(name),
value,
list(validate_namespace_path(namespace_path)),
)
async def list_secrets(self) -> List[str]:
async def list_secrets(
self, *, namespace_path: Optional[List[str]] = None
) -> List[str]:
"""The names of every Secret in this database. Names only."""
return await self._inner.list_secrets()
return await self._inner.list_secrets(
list(validate_namespace_path(namespace_path))
)
async def drop_secret(self, name: str) -> None:
async def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Drop a Secret. Bound Functions fail at their next job."""
await self._inner.drop_secret(validate_secret_name(name))
await self._inner.drop_secret(
validate_secret_name(name), list(validate_namespace_path(namespace_path))
)
async def describe_secret(self, name: str) -> SecretInfo:
async def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
"""What this database records about a Secret. Never the value."""
return SecretInfo.from_json(
await self._inner.describe_secret(validate_secret_name(name))
await self._inner.describe_secret(
validate_secret_name(name),
list(validate_namespace_path(namespace_path)),
)
)
async def list_jobs(self) -> List[JobInfo]:
python/python/lancedb/functions.py
+23 -3
View File
@@ -227,6 +227,19 @@ class FunctionOutput(_OpenRemoteValue):
fields: tuple[FunctionResultField, ...] = ()
class SecretReference(_RemoteValue):
"""Where a Secret lives, carried as its parts rather than as one string.
A joined id would need a delimiter, and a delimiter has to be excluded from
every name and segment forever, agreed on by both sides, and re-agreed each
time either grows a new way to be configured. Naming the parts settles all
of that: nothing here is parsed, so nothing can parse two ways.
"""
name: str
namespace_path: tuple[str, ...] = ()
class FunctionSignature(_RemoteValue):
inputs: tuple[FunctionParameter, ...]
output: FunctionOutput
@@ -310,7 +323,7 @@ class FunctionVersion(_RemoteValue):
runtime: PythonRuntimeSpec
runtime_digest: str
environment_digest: str
secret_env_bindings: Mapping[str, str] = {}
secret_env_bindings: Mapping[str, SecretReference] = {}
created_at: str
def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -383,7 +396,7 @@ class FunctionRegistrationRequest(_RemoteValue):
artifact: FunctionArtifactRequest
signature: FunctionSignature
runtime: PythonRuntimeSpec
secret_env_bindings: Mapping[str, str] = {}
secret_env_bindings: Mapping[str, SecretReference] = {}
class FunctionVersionRef(_OpenRemoteValue):
@@ -1322,7 +1335,14 @@ class UdfDefinition:
)
if not bindings:
return self._request
resolved = {binding.env_variable: binding.secret for binding in bindings}
# The binding records the full id -- path plus name -- because that is
# what the service resolves. At the root it is the bare name.
resolved = {
binding.env_variable: SecretReference(
name=binding.secret, namespace_path=tuple(binding.namespace_path)
)
for binding in bindings
}
return self._request._copy(update={"secret_env_bindings": resolved})
def __call__(self, *args, **kwargs):
python/python/lancedb/remote/db.py
+18 -10
View File
@@ -778,24 +778,32 @@ class RemoteDBConnection(DBConnection):
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def create_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.create_secret(name, value))
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path))
@override
def alter_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.alter_secret(name, value))
def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
@override
def describe_secret(self, name: str) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name))
def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
@override
def list_secrets(self) -> List[str]:
return LOOP.run(self._conn.list_secrets())
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
@override
def drop_secret(self, name: str) -> None:
LOOP.run(self._conn.drop_secret(name))
def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
@override
def list_jobs(self) -> List["JobInfo"]:
python/python/lancedb/secrets.py
+43 -4
View File
@@ -31,6 +31,31 @@ def validate_secret_name(name: str) -> str:
return name
def validate_namespace_path(namespace_path=None):
"""Check a namespace path locally and return it as a tuple.
``None`` and ``[]`` both mean the root namespace. Segments follow the same
rule as Secret names: a binding carries the path and the name as separate
fields, so neither is ever parsed out of the other.
"""
if namespace_path is None:
return ()
if isinstance(namespace_path, str):
raise TypeError(
"namespace_path must be a list of segments, not a string; "
f"did you mean [{namespace_path!r}]?"
)
segments = tuple(namespace_path)
for segment in segments:
if not isinstance(segment, str):
raise TypeError(
f"namespace path segment must be a string, not {type(segment).__name__}"
)
if not _SECRET_NAME.fullmatch(segment):
raise ValueError(f"invalid namespace path segment: {segment!r}")
return segments
def validate_env_variable(name: str) -> str:
"""Check an environment variable name locally and return it unchanged."""
if not isinstance(name, str):
@@ -76,11 +101,12 @@ class EnvVarSecret:
('openai-prod', 'OPENAI_API_KEY')
"""
__slots__ = ("_secret", "_env_variable")
__slots__ = ("_secret", "_env_variable", "_namespace_path")
def __init__(self, secret: str, env_variable: str):
def __init__(self, secret: str, env_variable: str, *, namespace_path=None):
self._secret = validate_secret_name(secret)
self._env_variable = validate_env_variable(env_variable)
self._namespace_path = validate_namespace_path(namespace_path)
@property
def secret(self) -> str:
@@ -92,10 +118,20 @@ class EnvVarSecret:
"""The environment variable the value is delivered in."""
return self._env_variable
@property
def namespace_path(self):
"""The namespace path the Secret is addressed within, root when empty."""
return list(self._namespace_path)
def __repr__(self) -> str:
path = (
f", namespace_path={list(self._namespace_path)!r}"
if self._namespace_path
else ""
)
return (
f"EnvVarSecret(secret={self._secret!r}, "
f"env_variable={self._env_variable!r})"
f"env_variable={self._env_variable!r}{path})"
)
def __eq__(self, other: object) -> bool:
@@ -103,10 +139,13 @@ class EnvVarSecret:
isinstance(other, EnvVarSecret)
and other._secret == self._secret
and other._env_variable == self._env_variable
and other._namespace_path == self._namespace_path
)
def __hash__(self) -> int:
return hash((EnvVarSecret, self._secret, self._env_variable))
return hash(
(EnvVarSecret, self._secret, self._env_variable, self._namespace_path)
)
class SecretInfo:
python/python/tests/test_first_class_function_slice1.py
+5 -2
View File
@@ -14,6 +14,7 @@ from lancedb.functions import (
FunctionVersion,
PythonRuntimeSpec,
RefreshColumnResult,
SecretReference,
)
from lancedb.table import AsyncTable
@@ -110,7 +111,9 @@ def test_function_version_identity_is_immutable_and_exact():
version = FunctionVersion.from_json(json.dumps(value))
assert version.name == "embed"
assert version.version == "fv_01K3EXACT"
assert dict(version.secret_env_bindings) == {"HF_TOKEN": "hf-prod"}
assert dict(version.secret_env_bindings) == {
"HF_TOKEN": SecretReference(name="hf-prod")
}
with pytest.raises((TypeError, ValueError)):
version.version = "fv_changed"
@@ -299,7 +302,7 @@ def test_canonical_client_values_carry_bindings_and_no_credentials():
json.dumps(job_result("remote_function_job.json"))
)
canonical = json.loads(version.to_canonical_json())
assert canonical["secret_env_bindings"] == {"HF_TOKEN": "hf-prod"}
assert canonical["secret_env_bindings"] == {"HF_TOKEN": {"name": "hf-prod"}}
assert_no_secret_values(canonical)
python/python/tests/test_first_class_function_slice2.py
+76 -5
View File
@@ -25,6 +25,7 @@ import lancedb
from lancedb.background_loop import LOOP
from lancedb.functions import (
PythonRuntimeSpec,
SecretReference,
UdfDefinition,
_canonical_arrow_type,
_GRAMMAR_PRIMITIVES,
@@ -94,6 +95,54 @@ def test_secret_bound_udf_matches_its_shared_registration_golden():
)
def test_a_namespaced_binding_records_the_path_and_the_name():
"""A binding names the parts, so nothing has to be parsed back out.
A root binding carries no path at all, which is what keeps its wire shape
identical to one written before namespaces existed.
"""
root = EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
assert root.namespace_path == []
nested = EnvVarSecret(
secret="openai-prod",
env_variable="OPENAI_API_KEY",
namespace_path=["prod", "vision"],
)
assert nested.namespace_path == ["prod", "vision"]
assert nested != root
bound = analyze_caption.bind_secrets([nested])
assert bound.secret_env_bindings == {
"OPENAI_API_KEY": SecretReference(
name="openai-prod", namespace_path=("prod", "vision")
)
}
at_root = analyze_caption.bind_secrets([root])
assert at_root.secret_env_bindings == {
"OPENAI_API_KEY": SecretReference(name="openai-prod")
}
canonical = json.loads(at_root.to_canonical_json())
assert canonical["secret_env_bindings"] == {
"OPENAI_API_KEY": {"name": "openai-prod"}
}
def test_a_namespace_path_is_validated_locally():
# The charset is the service's, not a delimiter's: a reference is never
# joined, so a segment cannot make anything parse two ways.
with pytest.raises(ValueError):
EnvVarSecret(
secret="openai-prod", env_variable="K", namespace_path=["with$delim"]
)
with pytest.raises(ValueError):
EnvVarSecret(secret="openai-prod", env_variable="K", namespace_path=["a/b"])
# A bare string is a plausible mistake with the wrong meaning.
with pytest.raises(TypeError):
EnvVarSecret(secret="openai-prod", env_variable="K", namespace_path="prod")
def test_an_unbound_request_carries_no_binding_at_all():
"""Binding is a registration-time decision, so the definition holds none.
@@ -167,7 +216,7 @@ def test_a_function_binds_at_most_sixteen_secrets():
envelope = json.loads(normalize_score.registration_request.to_canonical_json())
envelope["secret_env_bindings"] = {
f"TOKEN_{index}": f"secret-{index}" for index in range(17)
f"TOKEN_{index}": {"name": f"secret-{index}"} for index in range(17)
}
async def submit_envelope():
@@ -194,8 +243,8 @@ def test_binding_names_are_validated_below_the_python_api():
)
envelope = json.loads(analyze_caption.registration_request.to_canonical_json())
envelope["secret_env_bindings"] = {
"BAD=NAME": "openai-prod",
"TOKEN_0": "secret-0",
"BAD=NAME": {"name": "openai-prod"},
"TOKEN_0": {"name": "secret-0"},
}
envelope["runtime"]["env"]["TOKEN_0"] = "public"
@@ -306,6 +355,24 @@ def test_a_secret_name_admits_what_a_namespace_name_does():
with pytest.raises(ValueError, match="invalid Secret name"):
EnvVarSecret(secret=name, env_variable="OPENAI_API_KEY")
# A namespace segment follows the same rule, and LanceDB already admits
# these shapes as namespace names -- so a Secret is addressable inside one.
for segment in [".hidden", "_internal", "-lead", "trailing."]:
binding = EnvVarSecret(
secret="openai-prod",
env_variable="OPENAI_API_KEY",
namespace_path=[segment],
)
assert binding.namespace_path == [segment]
for segment in ["", "with/slash", "with$delimiter"]:
with pytest.raises(ValueError, match="invalid namespace path segment"):
EnvVarSecret(
secret="openai-prod",
env_variable="OPENAI_API_KEY",
namespace_path=[segment],
)
def _main_udf_source(
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
@@ -1579,10 +1646,14 @@ def test_remote_registration_sends_bindings_and_never_a_credential():
secrets=[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")],
)
assert dict(created.secret_env_bindings) == {"OPENAI_API_KEY": "openai-prod"}
assert dict(created.secret_env_bindings) == {
"OPENAI_API_KEY": SecretReference(name="openai-prod")
}
path, create_request = state["requests"][0]
assert path == "/v1/functions/create"
assert create_request["secret_env_bindings"] == {"OPENAI_API_KEY": "openai-prod"}
assert create_request["secret_env_bindings"] == {
"OPENAI_API_KEY": {"name": "openai-prod"}
}
# The request names a Secret and carries nothing that could be one.
assert create_request == json.loads(
analyze_caption.bind_secrets(
python/src/connection.rs
+30 -8
View File
@@ -708,10 +708,14 @@ impl Connection {
self_: PyRef<'_, Self>,
name: String,
value: String,
namespace_path: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.create_secret(name, value).await.infer_error()
inner
.create_secret(name, value, &namespace_path)
.await
.infer_error()
})
}
@@ -719,33 +723,51 @@ impl Connection {
self_: PyRef<'_, Self>,
name: String,
value: String,
namespace_path: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.alter_secret(name, value).await.infer_error()
inner
.alter_secret(name, value, &namespace_path)
.await
.infer_error()
})
}
pub fn list_secrets(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
pub fn list_secrets(
self_: PyRef<'_, Self>,
namespace_path: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.list_secrets().await.infer_error()
inner.list_secrets(&namespace_path).await.infer_error()
})
}
pub fn drop_secret(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
pub fn drop_secret(
self_: PyRef<'_, Self>,
name: String,
namespace_path: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
inner.drop_secret(name).await.infer_error()
inner.drop_secret(name, &namespace_path).await.infer_error()
})
}
/// Name and timestamps as a plain mapping. `SecretInfo` carries no value,
/// so there is none to filter out here.
pub fn describe_secret(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
pub fn describe_secret(
self_: PyRef<'_, Self>,
name: String,
namespace_path: Vec<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let info = inner.describe_secret(name).await.infer_error()?;
let info = inner
.describe_secret(name, &namespace_path)
.await
.infer_error()?;
Ok(HashMap::from([
("name".to_string(), info.name),
("created_at".to_string(), info.created_at),
rust/lancedb/src/connection.rs
+36 -10
View File
@@ -656,10 +656,17 @@ impl Connection {
/// rotation. There is no API that reads a stored credential back; the only
/// consumer is a Function that binds the Secret by name. Local databases
/// return [`Error::NotSupported`].
pub async fn create_secret(&self, name: impl AsRef<str>, value: impl AsRef<str>) -> Result<()> {
pub async fn create_secret(
&self,
name: impl AsRef<str>,
value: impl AsRef<str>,
namespace_path: &[String],
) -> Result<()> {
let value = value.as_ref();
crate::function::validate_secret_value(value)?;
self.internal.create_secret(name.as_ref(), value).await
self.internal
.create_secret(name.as_ref(), value, namespace_path)
.await
}
/// Replace the credential behind an existing Secret.
@@ -669,10 +676,17 @@ impl Connection {
/// minted -- which is what lets a rotation reach columns pinned to a
/// version registered before it. Local databases return
/// [`Error::NotSupported`].
pub async fn alter_secret(&self, name: impl AsRef<str>, value: impl AsRef<str>) -> Result<()> {
pub async fn alter_secret(
&self,
name: impl AsRef<str>,
value: impl AsRef<str>,
namespace_path: &[String],
) -> Result<()> {
let value = value.as_ref();
crate::function::validate_secret_value(value)?;
self.internal.alter_secret(name.as_ref(), value).await
self.internal
.alter_secret(name.as_ref(), value, namespace_path)
.await
}
/// The names of every Secret in this database.
@@ -680,8 +694,8 @@ impl Connection {
/// Names only. No path in this API returns a stored credential, by
/// construction rather than by policy. Local databases return
/// [`Error::NotSupported`].
pub async fn list_secrets(&self) -> Result<Vec<String>> {
self.internal.list_secrets().await
pub async fn list_secrets(&self, namespace_path: &[String]) -> Result<Vec<String>> {
self.internal.list_secrets(namespace_path).await
}
/// Drop a Secret.
@@ -690,8 +704,14 @@ impl Connection {
/// is the revocation path. The name becomes free to reuse, and a new
/// Secret under it is picked up by everything still bound to that name.
/// Local databases return [`Error::NotSupported`].
pub async fn drop_secret(&self, name: impl AsRef<str>) -> Result<()> {
self.internal.drop_secret(name.as_ref()).await
pub async fn drop_secret(
&self,
name: impl AsRef<str>,
namespace_path: &[String],
) -> Result<()> {
self.internal
.drop_secret(name.as_ref(), namespace_path)
.await
}
/// What this database records about one Secret: its name and timestamps.
@@ -699,8 +719,14 @@ impl Connection {
/// Never the value. The type it returns has no field for one, so this is a
/// property of the API rather than of what the caller chooses to read.
/// Local databases return [`Error::NotSupported`].
pub async fn describe_secret(&self, name: impl AsRef<str>) -> Result<SecretInfo> {
self.internal.describe_secret(name.as_ref()).await
pub async fn describe_secret(
&self,
name: impl AsRef<str>,
namespace_path: &[String],
) -> Result<SecretInfo> {
self.internal
.describe_secret(name.as_ref(), namespace_path)
.await
}
/// Rename a table in the database.
rust/lancedb/src/database.rs
+15 -5
View File
@@ -339,30 +339,40 @@ pub trait Database:
}
/// Create a named Secret in this database. Fails if the name is taken, so
/// a create can never silently become a rotation.
async fn create_secret(&self, _name: &str, _value: &str) -> Result<()> {
async fn create_secret(
&self,
_name: &str,
_value: &str,
_namespace_path: &[String],
) -> Result<()> {
secret_catalog_not_supported()
}
/// Replace the credential behind an existing Secret. Fails if it does not
/// exist. Every Function bound to it resolves the new value from its next
/// execution, with no new Function version.
async fn alter_secret(&self, _name: &str, _value: &str) -> Result<()> {
async fn alter_secret(
&self,
_name: &str,
_value: &str,
_namespace_path: &[String],
) -> Result<()> {
secret_catalog_not_supported()
}
/// The names of every Secret in this database.
///
/// Names only. No API path returns a stored credential, by construction
/// rather than by policy.
async fn list_secrets(&self) -> Result<Vec<String>> {
async fn list_secrets(&self, _namespace_path: &[String]) -> Result<Vec<String>> {
secret_catalog_not_supported()
}
/// Drop a Secret. Functions bound to it fail at their next job, which is
/// the revocation path.
async fn drop_secret(&self, _name: &str) -> Result<()> {
async fn drop_secret(&self, _name: &str, _namespace_path: &[String]) -> Result<()> {
secret_catalog_not_supported()
}
/// What the database records about one Secret: its name and timestamps,
/// never its value.
async fn describe_secret(&self, _name: &str) -> Result<SecretInfo> {
async fn describe_secret(&self, _name: &str, _namespace_path: &[String]) -> Result<SecretInfo> {
secret_catalog_not_supported()
}
/// Open a job by id, returning a handle with its record already
rust/lancedb/src/function.rs
+75 -4
View File
@@ -410,7 +410,7 @@ pub struct FunctionVersion {
runtime_digest: String,
environment_digest: String,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
secret_env_bindings: BTreeMap<String, String>,
secret_env_bindings: BTreeMap<String, SecretReference>,
created_at: String,
}
@@ -449,7 +449,7 @@ impl FunctionVersion {
/// them are not, and resolve at execution. Rotating a bound Secret
/// therefore changes what the same version runs with, and no value has a
/// field in this model.
pub fn secret_env_bindings(&self) -> &BTreeMap<String, String> {
pub fn secret_env_bindings(&self) -> &BTreeMap<String, SecretReference> {
&self.secret_env_bindings
}
@@ -499,6 +499,76 @@ pub struct FunctionArtifactRequest {
/// the count needs a bound for the same reason a credential needs a size limit.
pub const MAX_FUNCTION_SECRET_ENV_BINDINGS: usize = 16;
/// Where a Secret lives, carried as its parts rather than as one string.
///
/// A joined id would need a delimiter, and a delimiter has to be excluded from
/// every name and segment forever, agreed on by both sides, and re-agreed each
/// time either grows a new way to be configured. Naming the parts costs one
/// object and settles all of that: nothing here is parsed, so nothing can parse
/// two ways.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SecretReference {
pub name: String,
/// The namespace holding the Secret. Empty is the root, and is omitted from
/// the wire so a root binding carries no trace of a feature it does not use.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub namespace_path: Vec<String>,
}
impl SecretReference {
/// A Secret in the root namespace.
pub fn new(name: impl Into<String>) -> Self {
Self {
name: name.into(),
namespace_path: Vec::new(),
}
}
/// A Secret in `namespace_path`.
pub fn in_namespace(name: impl Into<String>, namespace_path: Vec<String>) -> Self {
Self {
name: name.into(),
namespace_path,
}
}
fn validate(&self) -> Result<()> {
validate_secret_component("Secret name", &self.name)?;
for segment in &self.namespace_path {
validate_secret_component("Secret namespace path segment", segment)?;
}
Ok(())
}
}
/// A Secret name or one namespace path segment.
///
/// Periods are legal here and delimiters are not a concern: a reference is
/// never joined into one string, so the only rule left is the character set the
/// service stores.
fn validate_secret_component(what: &str, value: &str) -> Result<()> {
if value.is_empty() || value.len() > MAX_SECRET_NAME_BYTES {
return Err(Error::InvalidInput {
message: format!(
"{what} must be 1..={MAX_SECRET_NAME_BYTES} bytes, got {}",
value.len()
),
});
}
if let Some(bad) = value
.chars()
.find(|c| !c.is_ascii_alphanumeric() && *c != '_' && *c != '-' && *c != '.')
{
return Err(Error::InvalidInput {
message: format!("{what} must match [A-Za-z0-9_.-], and {bad:?} does not"),
});
}
Ok(())
}
/// Longest Secret name or namespace path segment, matching the service.
pub const MAX_SECRET_NAME_BYTES: usize = 255;
/// Largest credential a Secret may hold, matching the limit the service
/// enforces. Bounded because the value is destined for a process environment.
pub const MAX_SECRET_VALUE_BYTES: usize = 64 * 1024;
@@ -558,7 +628,7 @@ pub struct FunctionRegistrationRequest {
/// a reference: whether the Secret exists is answered when a column is
/// declared against this version, not here.
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub secret_env_bindings: BTreeMap<String, String>,
pub secret_env_bindings: BTreeMap<String, SecretReference>,
}
impl FunctionRegistrationRequest {
@@ -576,7 +646,8 @@ impl FunctionRegistrationRequest {
),
});
}
for variable in self.secret_env_bindings.keys() {
for (variable, secret) in &self.secret_env_bindings {
secret.validate()?;
if !is_portable_env_name(variable) {
return Err(Error::InvalidInput {
message: format!(
rust/lancedb/src/remote/db.rs
+84 -23
View File
@@ -277,6 +277,22 @@ pub struct RemoteHostOverrides {
pub sql: Option<String>,
}
/// Attach a namespace path to a Secret request body.
///
/// A root path is omitted rather than sent empty, so a root request is byte
/// identical to one from a client that predates namespace addressing.
fn add_namespace_path(body: &mut serde_json::Value, namespace_path: &[String]) {
if namespace_path.is_empty() {
return;
}
body["namespace_path"] = serde_json::Value::Array(
namespace_path
.iter()
.map(|segment| serde_json::Value::String(segment.clone()))
.collect(),
);
}
impl RemoteDatabase {
pub(crate) fn try_new(
uri: &str,
@@ -356,11 +372,19 @@ impl<S: HttpSend> RemoteDatabase<S> {
/// requires, so they share one request shape. The value is a request field
/// and never a path segment or query parameter, which keeps it out of
/// access logs and proxy traces.
async fn write_secret(&self, route: &str, name: &str, value: &str) -> Result<()> {
let req = self.client.post(route).json(&serde_json::json!({
async fn write_secret(
&self,
route: &str,
name: &str,
value: &str,
namespace_path: &[String],
) -> Result<()> {
let mut body = serde_json::json!({
"name": name,
"value": value,
}));
});
add_namespace_path(&mut body, namespace_path);
let req = self.client.post(route).json(&body);
let (request_id, response) = self.client.send(req).await?;
self.client.check_response(&request_id, response).await?;
Ok(())
@@ -700,15 +724,22 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
Ok(response.dropped)
}
async fn create_secret(&self, name: &str, value: &str) -> Result<()> {
self.write_secret("/v1/secrets/create", name, value).await
async fn create_secret(
&self,
name: &str,
value: &str,
namespace_path: &[String],
) -> Result<()> {
self.write_secret("/v1/secrets/create", name, value, namespace_path)
.await
}
async fn alter_secret(&self, name: &str, value: &str) -> Result<()> {
self.write_secret("/v1/secrets/alter", name, value).await
async fn alter_secret(&self, name: &str, value: &str, namespace_path: &[String]) -> Result<()> {
self.write_secret("/v1/secrets/alter", name, value, namespace_path)
.await
}
async fn list_secrets(&self) -> Result<Vec<String>> {
async fn list_secrets(&self, namespace_path: &[String]) -> Result<Vec<String>> {
let mut names = Vec::new();
let mut page_token: Option<String> = None;
let mut seen_page_tokens = HashSet::new();
@@ -717,6 +748,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
if let Some(token) = &page_token {
body["page_token"] = serde_json::Value::String(token.clone());
}
add_namespace_path(&mut body, namespace_path);
let req = self.client.post("/v1/secrets/list").json(&body);
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
@@ -740,21 +772,19 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
Ok(names)
}
async fn drop_secret(&self, name: &str) -> Result<()> {
let req = self
.client
.post("/v1/secrets/drop")
.json(&serde_json::json!({ "name": name }));
async fn drop_secret(&self, name: &str, namespace_path: &[String]) -> Result<()> {
let mut body = serde_json::json!({ "name": name });
add_namespace_path(&mut body, namespace_path);
let req = self.client.post("/v1/secrets/drop").json(&body);
let (request_id, response) = self.client.send(req).await?;
self.client.check_response(&request_id, response).await?;
Ok(())
}
async fn describe_secret(&self, name: &str) -> Result<SecretInfo> {
let req = self
.client
.post("/v1/secrets/describe")
.json(&serde_json::json!({ "name": name }));
async fn describe_secret(&self, name: &str, namespace_path: &[String]) -> Result<SecretInfo> {
let mut body = serde_json::json!({ "name": name });
add_namespace_path(&mut body, namespace_path);
let req = self.client.post("/v1/secrets/describe").json(&body);
let (request_id, response) = self.client.send(req).await?;
let response = self.client.check_response(&request_id, response).await?;
response.json().await.err_to_http(request_id)
@@ -2886,11 +2916,11 @@ mod tests {
http::Response::builder().status(200).body("{}").unwrap()
});
if call {
conn.create_secret("openai-prod", "sk-live-0001")
conn.create_secret("openai-prod", "sk-live-0001", &[])
.await
.unwrap();
} else {
conn.alter_secret("openai-prod", "sk-live-0001")
conn.alter_secret("openai-prod", "sk-live-0001", &[])
.await
.unwrap();
}
@@ -2912,7 +2942,7 @@ mod tests {
http::Response::builder().status(200).body(body).unwrap()
});
assert_eq!(
conn.list_secrets().await.unwrap(),
conn.list_secrets(&[]).await.unwrap(),
vec!["openai-prod".to_string(), "hf-prod".to_string()]
);
}
@@ -2927,7 +2957,7 @@ mod tests {
.body(r#"{"secrets":[{"name":"openai-prod"}],"page_token":"same"}"#)
.unwrap()
});
let error = conn.list_secrets().await.unwrap_err();
let error = conn.list_secrets(&[]).await.unwrap_err();
assert!(
error.to_string().contains("repeated a page_token"),
"{error}"
@@ -2943,7 +2973,38 @@ mod tests {
assert_eq!(body, serde_json::json!({"name": "openai-prod"}));
http::Response::builder().status(200).body("{}").unwrap()
});
conn.drop_secret("openai-prod").await.unwrap();
conn.drop_secret("openai-prod", &[]).await.unwrap();
}
/// A namespace path is sent when there is one and omitted when there is
/// not, so a root request stays byte identical to one from a client that
/// predates namespace addressing -- which is what lets the parameter ship
/// before every server implements it.
#[tokio::test]
async fn test_a_namespace_path_is_sent_only_when_it_is_not_root() {
let conn = Connection::new_with_handler(|request| {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
body,
serde_json::json!({
"name": "openai-prod",
"namespace_path": ["prod", "vision"],
})
);
http::Response::builder().status(200).body("{}").unwrap()
});
conn.drop_secret("openai-prod", &["prod".to_string(), "vision".to_string()])
.await
.unwrap();
let conn = Connection::new_with_handler(|request| {
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert!(body.get("namespace_path").is_none(), "{body}");
http::Response::builder().status(200).body("{}").unwrap()
});
conn.drop_secret("openai-prod", &[]).await.unwrap();
}
#[tokio::test]
rust/lancedb/tests/first_class_function_slice1.rs
+3 -3
View File
@@ -6,7 +6,7 @@ use std::fs;
use std::path::PathBuf;
use lancedb::function::{
FunctionApplication, FunctionBinding, FunctionVersion, RefreshColumnResult,
FunctionApplication, FunctionBinding, FunctionVersion, RefreshColumnResult, SecretReference,
};
use serde_json::Value;
@@ -51,7 +51,7 @@ fn function_version_job_result_matches_shared_canonical_golden() {
assert_eq!(version.runtime_digest(), "sha256:runtime");
assert_eq!(
version.secret_env_bindings(),
&BTreeMap::from([("HF_TOKEN".to_string(), "hf-prod".to_string())])
&BTreeMap::from([("HF_TOKEN".to_string(), SecretReference::new("hf-prod"))])
);
assert_eq!(
version.to_canonical_json().expect("canonical JSON"),
@@ -181,7 +181,7 @@ fn canonical_client_values_carry_bindings_and_no_credentials() {
assert_eq!(
canonical["secret_env_bindings"],
serde_json::json!({"HF_TOKEN": "hf-prod"})
serde_json::json!({"HF_TOKEN": {"name": "hf-prod"}})
);
assert_no_secret_values(&canonical);
}
rust/lancedb/tests/first_class_function_slice2.rs
+13 -6
View File
@@ -7,6 +7,7 @@ use std::path::PathBuf;
use lancedb::Error;
use lancedb::function::{
FunctionRegistrationRequest, MAX_FUNCTION_SECRET_ENV_BINDINGS, MAX_SECRET_VALUE_BYTES,
SecretReference,
};
use serde_json::Value;
@@ -72,7 +73,7 @@ fn secret_bound_registration_request_matches_shared_canonical_golden() {
request.secret_env_bindings,
std::collections::BTreeMap::from([(
"OPENAI_API_KEY".to_string(),
"openai-prod".to_string()
SecretReference::new("openai-prod")
)])
);
assert_eq!(
@@ -131,7 +132,12 @@ async fn a_function_binds_at_most_sixteen_secrets() {
))
.unwrap();
request.secret_env_bindings = (0..=MAX_FUNCTION_SECRET_ENV_BINDINGS)
.map(|index| (format!("TOKEN_{index}"), format!("secret-{index}")))
.map(|index| {
(
format!("TOKEN_{index}"),
SecretReference::new(format!("secret-{index}")),
)
})
.collect();
let error = connection.create_function_async(request).await.unwrap_err();
@@ -156,7 +162,8 @@ async fn binding_names_are_validated_before_dispatch() {
"remote_function_registration_request.json",
))
.unwrap();
invalid_name.secret_env_bindings = [("BAD=NAME".to_string(), "openai-prod".to_string())].into();
invalid_name.secret_env_bindings =
[("BAD=NAME".to_string(), SecretReference::new("openai-prod"))].into();
let error = connection
.create_function_async(invalid_name)
.await
@@ -177,7 +184,7 @@ async fn binding_names_are_validated_before_dispatch() {
.env()
.and_then(|env| env.keys().next().cloned())
.expect("fixture runtime declares env");
overlapping.secret_env_bindings = [(bound.clone(), "openai-prod".to_string())].into();
overlapping.secret_env_bindings = [(bound.clone(), SecretReference::new("openai-prod"))].into();
let error = connection
.create_function_async(overlapping)
.await
@@ -200,7 +207,7 @@ async fn an_oversized_secret_value_is_refused_before_the_wire() {
for value in ["", &"x".repeat(MAX_SECRET_VALUE_BYTES + 1)] {
let error = connection
.create_secret("openai-prod", value)
.create_secret("openai-prod", value, &[])
.await
.unwrap_err();
assert!(
@@ -212,7 +219,7 @@ async fn an_oversized_secret_value_is_refused_before_the_wire() {
// A local database refuses the verb outright, which is what proves the
// size check ran ahead of the backend rather than instead of it.
let error = connection
.create_secret("openai-prod", "x".repeat(MAX_SECRET_VALUE_BYTES))
.create_secret("openai-prod", "x".repeat(MAX_SECRET_VALUE_BYTES), &[])
.await
.unwrap_err();
assert!(matches!(error, Error::NotSupported { .. }));
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json
Vendored
+32 -7
View File
@@ -3,7 +3,9 @@
"job_type": "create_function",
"job_state": "DONE",
"creation_ms": 1787270400000,
"spec": {"name": "embed"},
"spec": {
"name": "embed"
},
"result": {
"name": "embed",
"version": "fv_01K3EXACT",
@@ -13,19 +15,42 @@
"entrypoint": "embed"
},
"signature": {
"inputs": [{"name": "text", "arrow_type": "utf8", "nullable": true}],
"output": {"kind": "scalar", "arrow_type": "list<float32>", "nullable": false}
"inputs": [
{
"name": "text",
"arrow_type": "utf8",
"nullable": true
}
],
"output": {
"kind": "scalar",
"arrow_type": "list<float32>",
"nullable": false
}
},
"runtime": {
"kind": "python",
"python_version": "3.12",
"environment": {"kind": "pip", "packages": ["sentence-transformers>=3"]},
"env": {"TOKENIZERS_PARALLELISM": "false"}
"environment": {
"kind": "pip",
"packages": [
"sentence-transformers>=3"
]
},
"env": {
"TOKENIZERS_PARALLELISM": "false"
}
},
"runtime_digest": "sha256:runtime",
"environment_digest": "sha256:environment",
"secret_env_bindings": {"HF_TOKEN": "hf-prod"},
"secret_env_bindings": {
"HF_TOKEN": {
"name": "hf-prod"
}
},
"created_at": "2026-08-21T00:00:00Z"
},
"future_job": {"trace_id": "trace-1"}
"future_job": {
"trace_id": "trace-1"
}
}
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json
Vendored
+1 -1
View File
@@ -1 +1 @@
{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_env_bindings":{"OPENAI_API_KEY":"openai-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}}
{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_env_bindings":{"OPENAI_API_KEY":{"name":"openai-prod"}},"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}}
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json
Vendored
+3 -1
View File
@@ -27,7 +27,9 @@
"python_version": "3.12"
},
"secret_env_bindings": {
"OPENAI_API_KEY": "openai-prod"
"OPENAI_API_KEY": {
"name": "openai-prod"
}
},
"signature": {
"inputs": [
rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json
Vendored
+1 -1
View File
@@ -1 +1 @@
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","secret_env_bindings":{"HF_TOKEN":"hf-prod"},"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}
{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","secret_env_bindings":{"HF_TOKEN":{"name":"hf-prod"}},"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list<float32>","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"}