feat(secrets): address Secrets by namespace path

A Secret is identified by a namespace path plus a name, and resolution is
exact: one under `["prod"]` is not visible from `["prod", "vision"]` and never
falls back to a parent.

The binding names it as a `SecretReference` of `{name, namespace_path}` rather
than one joined string. A joined id needs a delimiter, and a delimiter has to be
excluded from every name and every segment forever, agreed on by both sides, and
re-agreed each time either side grows a new way to be configured --
`ClientConfig.id_delimiter` is exactly that, configurable and advertised on
every request, so an id joined on a fixed `$` would contradict a delimiter the
same request declares. Naming the parts costs one object and settles it.

Every verb takes `namespace_path` keyword-only, defaulting to the root. On
`EnvVarSecret` the same argument is `secret_namespace_path`, matching
`secret_name`: a binding names two things, and the unqualified word reads as
the variable's namespace rather than the Secret's. It pins the path at
construction, so a worker resolves the Secret it was handed rather than
re-resolving against its own default.

The five PyO3 secret methods take `Option<Vec<String>>` with a `None` default,
as every other namespace-taking method on `Connection` already does. They were
the only ones requiring the argument positionally, so the keyword form their
neighbours accept did not work on them.

A root path is omitted from the request body rather than sent empty, so a root
request is byte-identical to one from a client that predates this, and a root
binding to one written before namespaces existed. Tests pin both.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UfmeJ533rQDnPBkMtjerV6
This commit is contained in:
Jonathan M Hsieh
2026-09-15 19:29:09 +00:00
co-authored by Claude Opus 5
parent fc68417397
commit dddc94474f
18 changed files with 511 additions and 112 deletions
+15 -5
View File
@@ -153,11 +153,21 @@ class Connection(object):
async def get_function(self, name: str, version: str) -> str: ...
async def list_functions(self) -> List[str]: ...
async def drop_function(self, name: str, version: str) -> bool: ...
async def create_secret(self, name: str, value: str) -> None: ...
async def alter_secret(self, name: str, value: str) -> None: ...
async def list_secrets(self) -> List[str]: ...
async def drop_secret(self, name: str) -> None: ...
async def describe_secret(self, name: str) -> Dict[str, str]: ...
async def create_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
) -> None: ...
async def alter_secret(
self, name: str, value: str, namespace_path: Optional[List[str]] = None
) -> None: ...
async def list_secrets(
self, namespace_path: Optional[List[str]] = None
) -> List[str]: ...
async def drop_secret(
self, name: str, namespace_path: Optional[List[str]] = None
) -> None: ...
async def describe_secret(
self, name: str, namespace_path: Optional[List[str]] = None
) -> Tuple[str, int, int]: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def cancel_job(self, job_id: str) -> bool: ...
async def execute_query_async(
+70 -26
View File
@@ -58,7 +58,12 @@ from .materialized_view import (
SelectArg,
normalize_select,
)
from .secrets import EnvVarSecret, SecretInfo, validate_secret_name
from .secrets import (
EnvVarSecret,
SecretInfo,
validate_namespace_path,
validate_secret_name,
)
from .table import (
AsyncTable,
LanceTable,
@@ -831,7 +836,9 @@ class DBConnection(EnforceOverrides):
"Function catalog operations are not supported for this connection type"
)
def create_secret(self, name: str, value: str) -> None:
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Create a named Secret in this database.
Fails if the name is taken, so a create never silently becomes a
@@ -843,7 +850,9 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def alter_secret(self, name: str, value: str) -> None:
def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Replace the credential behind an existing Secret.
Fails if it does not exist. Every Function bound to the Secret uses the
@@ -855,7 +864,7 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def list_secrets(self) -> List[str]:
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
"""The names of every Secret in this database.
Names only. No method returns a stored credential, by construction
@@ -865,7 +874,9 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def drop_secret(self, name: str) -> None:
def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Drop a Secret.
Functions bound to it fail at their next job, naming the Secret; that
@@ -877,7 +888,9 @@ class DBConnection(EnforceOverrides):
"Secret operations are not supported for this connection type"
)
def describe_secret(self, name: str) -> SecretInfo:
def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
"""What this database records about a Secret: name and timestamps.
Never the value -- there is no code path that could return one. Local
@@ -1668,24 +1681,32 @@ class LanceDBConnection(DBConnection):
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def create_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.create_secret(name, value))
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path))
@override
def alter_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.alter_secret(name, value))
def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
@override
def list_secrets(self) -> List[str]:
return LOOP.run(self._conn.list_secrets())
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
@override
def drop_secret(self, name: str) -> None:
LOOP.run(self._conn.drop_secret(name))
def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
@override
def describe_secret(self, name: str) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name))
def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
@override
def list_jobs(self) -> List[JobInfo]:
@@ -2574,34 +2595,57 @@ class AsyncConnection(object):
"""Drop one exact immutable Function version from the remote catalog."""
return await self._inner.drop_function(name, version)
async def create_secret(self, name: str, value: str) -> None:
async def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Create a named Secret in this database.
Fails if the name is taken, so a create never silently becomes a
rotation. Nothing reads the value back.
"""
await self._inner.create_secret(validate_secret_name(name), value)
await self._inner.create_secret(
validate_secret_name(name),
value,
list(validate_namespace_path(namespace_path)),
)
async def alter_secret(self, name: str, value: str) -> None:
async def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Replace the credential behind an existing Secret.
Fails if it does not exist. Bound Functions use the new value from
their next job, with no new Function version.
"""
await self._inner.alter_secret(validate_secret_name(name), value)
await self._inner.alter_secret(
validate_secret_name(name),
value,
list(validate_namespace_path(namespace_path)),
)
async def list_secrets(self) -> List[str]:
async def list_secrets(
self, *, namespace_path: Optional[List[str]] = None
) -> List[str]:
"""The names of every Secret in this database. Names only."""
return await self._inner.list_secrets()
return await self._inner.list_secrets(
list(validate_namespace_path(namespace_path))
)
async def drop_secret(self, name: str) -> None:
async def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
"""Drop a Secret. Bound Functions fail at their next job."""
await self._inner.drop_secret(validate_secret_name(name))
await self._inner.drop_secret(
validate_secret_name(name), list(validate_namespace_path(namespace_path))
)
async def describe_secret(self, name: str) -> SecretInfo:
async def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
"""What this database records about a Secret. Never the value."""
name, created_at_millis, updated_at_millis = await self._inner.describe_secret(
validate_secret_name(name)
validate_secret_name(name),
list(validate_namespace_path(namespace_path)),
)
return SecretInfo(
name=name,
+18 -2
View File
@@ -227,6 +227,19 @@ class FunctionOutput(_OpenRemoteValue):
fields: tuple[FunctionResultField, ...] = ()
class SecretReference(_RemoteValue):
"""Where a Secret lives, carried as its parts rather than as one string.
A joined id would need a delimiter, and a delimiter has to be excluded from
every name and segment forever, agreed on by both sides, and re-agreed each
time either grows a new way to be configured. Naming the parts settles all
of that: nothing here is parsed, so nothing can parse two ways.
"""
name: str
namespace_path: tuple[str, ...] = ()
class SecretBinding(_RemoteValue):
"""How a Secret reaches the Function that binds it.
@@ -238,7 +251,7 @@ class SecretBinding(_RemoteValue):
kind: str
variable: Optional[str] = None
secret_ref: Optional[str] = None
secret_ref: Optional[SecretReference] = None
class FunctionSignature(_RemoteValue):
@@ -1344,7 +1357,10 @@ class UdfDefinition:
SecretBinding(
kind="env",
variable=binding.env_variable,
secret_ref=binding.secret_name,
secret_ref=SecretReference(
name=binding.secret_name,
namespace_path=tuple(binding.secret_namespace_path),
),
)
for binding in bindings
),
+18 -10
View File
@@ -836,24 +836,32 @@ class RemoteDBConnection(DBConnection):
return LOOP.run(self._conn.drop_function(name, version=version))
@override
def create_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.create_secret(name, value))
def create_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path))
@override
def alter_secret(self, name: str, value: str) -> None:
LOOP.run(self._conn.alter_secret(name, value))
def alter_secret(
self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
@override
def describe_secret(self, name: str) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name))
def describe_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> SecretInfo:
return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
@override
def list_secrets(self) -> List[str]:
return LOOP.run(self._conn.list_secrets())
def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
@override
def drop_secret(self, name: str) -> None:
LOOP.run(self._conn.drop_secret(name))
def drop_secret(
self, name: str, *, namespace_path: Optional[List[str]] = None
) -> None:
LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
@override
def list_jobs(self) -> List["JobInfo"]:
+54 -4
View File
@@ -31,6 +31,31 @@ def validate_secret_name(name: str) -> str:
return name
def validate_namespace_path(namespace_path=None):
"""Check a namespace path locally and return it as a tuple.
``None`` and ``[]`` both mean the root namespace. Segments follow the same
rule as Secret names: a binding carries the path and the name as separate
fields, so neither is ever parsed out of the other.
"""
if namespace_path is None:
return ()
if isinstance(namespace_path, str):
raise TypeError(
"namespace_path must be a list of segments, not a string; "
f"did you mean [{namespace_path!r}]?"
)
segments = tuple(namespace_path)
for segment in segments:
if not isinstance(segment, str):
raise TypeError(
f"namespace path segment must be a string, not {type(segment).__name__}"
)
if not _SECRET_NAME.fullmatch(segment):
raise ValueError(f"invalid namespace path segment: {segment!r}")
return segments
def validate_env_variable(name: str) -> str:
"""Check an environment variable name locally and return it unchanged."""
if not isinstance(name, str):
@@ -67,6 +92,10 @@ class EnvVarSecret:
The Secret's database-scoped name.
env_variable : str
The environment variable the Function reads it from.
secret_namespace_path : list of str, optional
The namespace the Secret is addressed within. ``None`` and ``[]`` both
mean the root namespace. Carried beside the name rather than joined
into it, so neither is ever parsed back out of the other.
Examples
--------
@@ -78,11 +107,14 @@ class EnvVarSecret:
('openai-prod', 'OPENAI_API_KEY')
"""
__slots__ = ("_secret_name", "_env_variable")
__slots__ = ("_secret_name", "_env_variable", "_secret_namespace_path")
def __init__(self, secret_name: str, env_variable: str):
def __init__(
self, secret_name: str, env_variable: str, *, secret_namespace_path=None
):
self._secret_name = validate_secret_name(secret_name)
self._env_variable = validate_env_variable(env_variable)
self._secret_namespace_path = validate_namespace_path(secret_namespace_path)
@property
def secret_name(self) -> str:
@@ -94,10 +126,20 @@ class EnvVarSecret:
"""The environment variable the value is delivered in."""
return self._env_variable
@property
def secret_namespace_path(self):
"""The namespace path the Secret is addressed within, root when empty."""
return list(self._secret_namespace_path)
def __repr__(self) -> str:
path = (
f", secret_namespace_path={list(self._secret_namespace_path)!r}"
if self._secret_namespace_path
else ""
)
return (
f"EnvVarSecret(secret_name={self._secret_name!r}, "
f"env_variable={self._env_variable!r})"
f"env_variable={self._env_variable!r}{path})"
)
def __eq__(self, other: object) -> bool:
@@ -105,10 +147,18 @@ class EnvVarSecret:
isinstance(other, EnvVarSecret)
and other._secret_name == self._secret_name
and other._env_variable == self._env_variable
and other._secret_namespace_path == self._secret_namespace_path
)
def __hash__(self) -> int:
return hash((EnvVarSecret, self._secret_name, self._env_variable))
return hash(
(
EnvVarSecret,
self._secret_name,
self._env_variable,
self._secret_namespace_path,
)
)
class SecretInfo:
@@ -15,6 +15,7 @@ from lancedb.functions import (
PythonRuntimeSpec,
SecretBinding,
RefreshColumnResult,
SecretReference,
)
from lancedb.table import AsyncTable
@@ -112,7 +113,9 @@ def test_function_version_identity_is_immutable_and_exact():
assert version.name == "embed"
assert version.version == "fv_01K3EXACT"
assert list(version.secret_bindings) == [
SecretBinding(kind="env", variable="HF_TOKEN", secret_ref="hf-prod")
SecretBinding(
kind="env", variable="HF_TOKEN", secret_ref=SecretReference(name="hf-prod")
)
]
with pytest.raises((TypeError, ValueError)):
@@ -303,7 +306,7 @@ def test_canonical_client_values_carry_bindings_and_no_credentials():
)
canonical = json.loads(version.to_canonical_json())
assert canonical["secret_bindings"] == [
{"kind": "env", "variable": "HF_TOKEN", "secret_ref": "hf-prod"}
{"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}}
]
assert_no_secret_values(canonical)
@@ -26,6 +26,7 @@ import lancedb
from lancedb.functions import (
PythonRuntimeSpec,
SecretBinding,
SecretReference,
UdfDefinition,
_canonical_arrow_type,
_GRAMMAR_PRIMITIVES,
@@ -95,6 +96,73 @@ def test_secret_bound_udf_matches_its_shared_registration_golden():
)
def test_a_namespaced_binding_records_the_path_and_the_name():
"""A binding names the parts, so nothing has to be parsed back out.
A root binding carries no path at all, which is what keeps its wire shape
identical to one written before namespaces existed.
"""
root = EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")
assert root.secret_namespace_path == []
nested = EnvVarSecret(
secret_name="openai-prod",
env_variable="OPENAI_API_KEY",
secret_namespace_path=["prod", "vision"],
)
assert nested.secret_namespace_path == ["prod", "vision"]
assert nested != root
bound = analyze_caption.bind_secrets([nested])
assert list(bound.secret_bindings) == [
SecretBinding(
kind="env",
variable="OPENAI_API_KEY",
secret_ref=SecretReference(
name="openai-prod", namespace_path=("prod", "vision")
),
)
]
at_root = analyze_caption.bind_secrets([root])
assert list(at_root.secret_bindings) == [
SecretBinding(
kind="env",
variable="OPENAI_API_KEY",
secret_ref=SecretReference(name="openai-prod"),
)
]
# A root binding carries no path at all on the wire.
canonical = json.loads(at_root.to_canonical_json())
assert canonical["secret_bindings"] == [
{
"kind": "env",
"variable": "OPENAI_API_KEY",
"secret_ref": {"name": "openai-prod"},
}
]
def test_a_namespace_path_is_validated_locally():
# The charset is the service's, not a delimiter's: a reference is never
# joined, so a segment cannot make anything parse two ways.
with pytest.raises(ValueError):
EnvVarSecret(
secret_name="openai-prod",
env_variable="K",
secret_namespace_path=["with$delim"],
)
with pytest.raises(ValueError):
EnvVarSecret(
secret_name="openai-prod", env_variable="K", secret_namespace_path=["a/b"]
)
# A bare string is a plausible mistake with the wrong meaning.
with pytest.raises(TypeError):
EnvVarSecret(
secret_name="openai-prod", env_variable="K", secret_namespace_path="prod"
)
def test_an_unbound_request_carries_no_binding_at_all():
"""Binding is a registration-time decision, so the definition holds none.
@@ -171,9 +239,11 @@ def test_a_binding_envelope_reaches_the_service_for_it_to_judge():
sent = state["requests"][0][1]
assert len(sent["secret_bindings"]) == 17
assert {"kind": "env", "variable": "TOKEN_0", "secret_ref": "secret-0"} in sent[
"secret_bindings"
]
assert {
"kind": "env",
"variable": "TOKEN_0",
"secret_ref": {"name": "secret-0"},
} in sent["secret_bindings"]
_SECRET_DEBUG_LOG_SOURCE = """
@@ -274,6 +344,24 @@ def test_a_secret_name_admits_what_a_namespace_name_does():
with pytest.raises(ValueError, match="invalid Secret name"):
EnvVarSecret(secret_name=name, env_variable="OPENAI_API_KEY")
# A namespace segment follows the same rule, and LanceDB already admits
# these shapes as namespace names -- so a Secret is addressable inside one.
for segment in [".hidden", "_internal", "-lead", "trailing."]:
binding = EnvVarSecret(
secret_name="openai-prod",
env_variable="OPENAI_API_KEY",
secret_namespace_path=[segment],
)
assert binding.secret_namespace_path == [segment]
for segment in ["", "with/slash", "with$delimiter"]:
with pytest.raises(ValueError, match="invalid namespace path segment"):
EnvVarSecret(
secret_name="openai-prod",
env_variable="OPENAI_API_KEY",
secret_namespace_path=[segment],
)
def _main_udf_source(
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
@@ -1568,12 +1656,20 @@ def test_remote_registration_sends_bindings_and_never_a_credential():
)
assert list(created.secret_bindings) == [
SecretBinding(kind="env", variable="OPENAI_API_KEY", secret_ref="openai-prod")
SecretBinding(
kind="env",
variable="OPENAI_API_KEY",
secret_ref=SecretReference(name="openai-prod"),
)
]
path, create_request = state["requests"][0]
assert path == "/v1/function/analyze_caption/create"
assert create_request["secret_bindings"] == [
{"kind": "env", "variable": "OPENAI_API_KEY", "secret_ref": "openai-prod"}
{
"kind": "env",
"variable": "OPENAI_API_KEY",
"secret_ref": {"name": "openai-prod"},
}
]
# The request names a Secret and carries nothing that could be one. The
# Function's own name is the path identifier rather than a body field, so
+40 -8
View File
@@ -768,49 +768,81 @@ impl Connection {
})
}
#[pyo3(signature = (name, value, namespace_path=None))]
pub fn create_secret(
self_: PyRef<'_, Self>,
name: String,
value: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.create_secret(name, value).await.infer_error()
inner
.create_secret(name, value, &namespace_path)
.await
.infer_error()
})
}
#[pyo3(signature = (name, value, namespace_path=None))]
pub fn alter_secret(
self_: PyRef<'_, Self>,
name: String,
value: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.alter_secret(name, value).await.infer_error()
inner
.alter_secret(name, value, &namespace_path)
.await
.infer_error()
})
}
pub fn list_secrets(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
#[pyo3(signature = (namespace_path=None))]
pub fn list_secrets(
self_: PyRef<'_, Self>,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.list_secrets().await.infer_error()
inner.list_secrets(&namespace_path).await.infer_error()
})
}
pub fn drop_secret(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
#[pyo3(signature = (name, namespace_path=None))]
pub fn drop_secret(
self_: PyRef<'_, Self>,
name: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
inner.drop_secret(name).await.infer_error()
inner.drop_secret(name, &namespace_path).await.infer_error()
})
}
/// Name and timestamps as a plain tuple. `SecretInfo` carries no value, so
/// there is none to filter out here. Timestamps stay integers rather than
/// going through a string, so the caller can compare two without parsing.
pub fn describe_secret(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
#[pyo3(signature = (name, namespace_path=None))]
pub fn describe_secret(
self_: PyRef<'_, Self>,
name: String,
namespace_path: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
let info = inner.describe_secret(name).await.infer_error()?;
let info = inner
.describe_secret(name, &namespace_path)
.await
.infer_error()?;
Ok((info.name, info.created_at_millis, info.updated_at_millis))
})
}