feat(secrets): address Secrets by namespace path

A binding names a Secret by its parts -- `{name, namespace_path}` -- rather
than by one string that has to be parsed back apart. A joined id needs a
delimiter, and a delimiter has to be excluded from every name and every segment
forever, agreed on by both sides, and re-agreed each time either grows a new way
to be configured. `ClientConfig.id_delimiter` is exactly that: it already
governs table identifiers and is advertised on every request, so an id joined
on a fixed `$` would contradict a delimiter the same request declares.

Naming the parts settles all of it at the cost of one object. Nothing is
parsed, so nothing can parse two ways; the charset is the service's storage
rule rather than a delimiter's shadow; and the Secret verbs already send
`namespace_path` as a field, so the binding now matches them instead of being
the one place that flattens.

A root binding omits the path entirely, so its wire shape is byte-identical to
one written before namespaces existed.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UfmeJ533rQDnPBkMtjerV6

python/python/lancedb/_lancedb.pyi

@@ -153,11 +153,17 @@ class Connection(object):
     async def get_function(self, name: str, version: str) -> str: ...
     async def list_functions(self) -> List[str]: ...
     async def drop_function(self, name: str, version: str) -> bool: ...
-    async def create_secret(self, name: str, value: str) -> None: ...
-    async def alter_secret(self, name: str, value: str) -> None: ...
-    async def list_secrets(self) -> List[str]: ...
-    async def drop_secret(self, name: str) -> None: ...
-    async def describe_secret(self, name: str) -> Dict[str, str]: ...
+    async def create_secret(
+        self, name: str, value: str, namespace_path: List[str]
+    ) -> None: ...
+    async def alter_secret(
+        self, name: str, value: str, namespace_path: List[str]
+    ) -> None: ...
+    async def list_secrets(self, namespace_path: List[str]) -> List[str]: ...
+    async def drop_secret(self, name: str, namespace_path: List[str]) -> None: ...
+    async def describe_secret(
+        self, name: str, namespace_path: List[str]
+    ) -> Dict[str, str]: ...
     async def list_jobs(self) -> List[JobInfo]: ...
     async def cancel_job(self, job_id: str) -> bool: ...
     async def execute_query_async(

python/python/lancedb/db.py

@@ -58,7 +58,12 @@ from .materialized_view import (
     SelectArg,
     normalize_select,
 )
-from .secrets import EnvVarSecret, SecretInfo, validate_secret_name
+from .secrets import (
+    EnvVarSecret,
+    SecretInfo,
+    validate_namespace_path,
+    validate_secret_name,
+)
 from .table import (
     AsyncTable,
     LanceTable,
@@ -779,7 +784,9 @@ class DBConnection(EnforceOverrides):
             "Function catalog operations are not supported for this connection type"
         )
 
-    def create_secret(self, name: str, value: str) -> None:
+    def create_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
         """Create a named Secret in this database.
 
         Fails if the name is taken, so a create never silently becomes a
@@ -791,7 +798,9 @@ class DBConnection(EnforceOverrides):
             "Secret operations are not supported for this connection type"
         )
 
-    def alter_secret(self, name: str, value: str) -> None:
+    def alter_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
         """Replace the credential behind an existing Secret.
 
         Fails if it does not exist. Every Function bound to the Secret uses the
@@ -803,7 +812,7 @@ class DBConnection(EnforceOverrides):
             "Secret operations are not supported for this connection type"
         )
 
-    def list_secrets(self) -> List[str]:
+    def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
         """The names of every Secret in this database.
 
         Names only. No method returns a stored credential, by construction
@@ -813,7 +822,9 @@ class DBConnection(EnforceOverrides):
             "Secret operations are not supported for this connection type"
         )
 
-    def drop_secret(self, name: str) -> None:
+    def drop_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
         """Drop a Secret.
 
         Functions bound to it fail at their next job, naming the Secret; that
@@ -825,7 +836,9 @@ class DBConnection(EnforceOverrides):
             "Secret operations are not supported for this connection type"
         )
 
-    def describe_secret(self, name: str) -> SecretInfo:
+    def describe_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> SecretInfo:
         """What this database records about a Secret: name and timestamps.
 
         Never the value -- there is no code path that could return one. Local
@@ -1569,24 +1582,32 @@ class LanceDBConnection(DBConnection):
         return LOOP.run(self._conn.drop_function(name, version=version))
 
     @override
-    def create_secret(self, name: str, value: str) -> None:
-        LOOP.run(self._conn.create_secret(name, value))
+    def create_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
+        LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path))
 
     @override
-    def alter_secret(self, name: str, value: str) -> None:
-        LOOP.run(self._conn.alter_secret(name, value))
+    def alter_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
+        LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
 
     @override
-    def list_secrets(self) -> List[str]:
-        return LOOP.run(self._conn.list_secrets())
+    def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
+        return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
 
     @override
-    def drop_secret(self, name: str) -> None:
-        LOOP.run(self._conn.drop_secret(name))
+    def drop_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
+        LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
 
     @override
-    def describe_secret(self, name: str) -> SecretInfo:
-        return LOOP.run(self._conn.describe_secret(name))
+    def describe_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> SecretInfo:
+        return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
 
     @override
     def list_jobs(self) -> List[JobInfo]:
@@ -2418,34 +2439,59 @@ class AsyncConnection(object):
         """Drop one exact immutable Function version from the remote catalog."""
         return await self._inner.drop_function(name, version)
 
-    async def create_secret(self, name: str, value: str) -> None:
+    async def create_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
         """Create a named Secret in this database.
 
         Fails if the name is taken, so a create never silently becomes a
         rotation. Nothing reads the value back.
         """
-        await self._inner.create_secret(validate_secret_name(name), value)
+        await self._inner.create_secret(
+            validate_secret_name(name),
+            value,
+            list(validate_namespace_path(namespace_path)),
+        )
 
-    async def alter_secret(self, name: str, value: str) -> None:
+    async def alter_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
         """Replace the credential behind an existing Secret.
 
         Fails if it does not exist. Bound Functions use the new value from
         their next job, with no new Function version.
         """
-        await self._inner.alter_secret(validate_secret_name(name), value)
+        await self._inner.alter_secret(
+            validate_secret_name(name),
+            value,
+            list(validate_namespace_path(namespace_path)),
+        )
 
-    async def list_secrets(self) -> List[str]:
+    async def list_secrets(
+        self, *, namespace_path: Optional[List[str]] = None
+    ) -> List[str]:
         """The names of every Secret in this database. Names only."""
-        return await self._inner.list_secrets()
+        return await self._inner.list_secrets(
+            list(validate_namespace_path(namespace_path))
+        )
 
-    async def drop_secret(self, name: str) -> None:
+    async def drop_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
         """Drop a Secret. Bound Functions fail at their next job."""
-        await self._inner.drop_secret(validate_secret_name(name))
+        await self._inner.drop_secret(
+            validate_secret_name(name), list(validate_namespace_path(namespace_path))
+        )
 
-    async def describe_secret(self, name: str) -> SecretInfo:
+    async def describe_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> SecretInfo:
         """What this database records about a Secret. Never the value."""
         return SecretInfo.from_json(
-            await self._inner.describe_secret(validate_secret_name(name))
+            await self._inner.describe_secret(
+                validate_secret_name(name),
+                list(validate_namespace_path(namespace_path)),
+            )
         )
 
     async def list_jobs(self) -> List[JobInfo]:

python/python/lancedb/functions.py

@@ -227,6 +227,19 @@ class FunctionOutput(_OpenRemoteValue):
     fields: tuple[FunctionResultField, ...] = ()
 
 
+class SecretReference(_RemoteValue):
+    """Where a Secret lives, carried as its parts rather than as one string.
+
+    A joined id would need a delimiter, and a delimiter has to be excluded from
+    every name and segment forever, agreed on by both sides, and re-agreed each
+    time either grows a new way to be configured. Naming the parts settles all
+    of that: nothing here is parsed, so nothing can parse two ways.
+    """
+
+    name: str
+    namespace_path: tuple[str, ...] = ()
+
+
 class FunctionSignature(_RemoteValue):
     inputs: tuple[FunctionParameter, ...]
     output: FunctionOutput
@@ -310,7 +323,7 @@ class FunctionVersion(_RemoteValue):
     runtime: PythonRuntimeSpec
     runtime_digest: str
     environment_digest: str
-    secret_env_bindings: Mapping[str, str] = {}
+    secret_env_bindings: Mapping[str, SecretReference] = {}
     created_at: str
 
     def __call__(self, **inputs: Any) -> FunctionApplication:
@@ -383,7 +396,7 @@ class FunctionRegistrationRequest(_RemoteValue):
     artifact: FunctionArtifactRequest
     signature: FunctionSignature
     runtime: PythonRuntimeSpec
-    secret_env_bindings: Mapping[str, str] = {}
+    secret_env_bindings: Mapping[str, SecretReference] = {}
 
 
 class FunctionVersionRef(_OpenRemoteValue):
@@ -1322,7 +1335,14 @@ class UdfDefinition:
             )
         if not bindings:
             return self._request
-        resolved = {binding.env_variable: binding.secret for binding in bindings}
+        # The binding records the full id -- path plus name -- because that is
+        # what the service resolves. At the root it is the bare name.
+        resolved = {
+            binding.env_variable: SecretReference(
+                name=binding.secret, namespace_path=tuple(binding.namespace_path)
+            )
+            for binding in bindings
+        }
         return self._request._copy(update={"secret_env_bindings": resolved})
 
     def __call__(self, *args, **kwargs):

python/python/lancedb/remote/db.py

@@ -778,24 +778,32 @@ class RemoteDBConnection(DBConnection):
         return LOOP.run(self._conn.drop_function(name, version=version))
 
     @override
-    def create_secret(self, name: str, value: str) -> None:
-        LOOP.run(self._conn.create_secret(name, value))
+    def create_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
+        LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path))
 
     @override
-    def alter_secret(self, name: str, value: str) -> None:
-        LOOP.run(self._conn.alter_secret(name, value))
+    def alter_secret(
+        self, name: str, value: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
+        LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path))
 
     @override
-    def describe_secret(self, name: str) -> SecretInfo:
-        return LOOP.run(self._conn.describe_secret(name))
+    def describe_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> SecretInfo:
+        return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path))
 
     @override
-    def list_secrets(self) -> List[str]:
-        return LOOP.run(self._conn.list_secrets())
+    def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]:
+        return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path))
 
     @override
-    def drop_secret(self, name: str) -> None:
-        LOOP.run(self._conn.drop_secret(name))
+    def drop_secret(
+        self, name: str, *, namespace_path: Optional[List[str]] = None
+    ) -> None:
+        LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path))
 
     @override
     def list_jobs(self) -> List["JobInfo"]:

python/python/lancedb/secrets.py

@@ -31,6 +31,31 @@ def validate_secret_name(name: str) -> str:
     return name
 
 
+def validate_namespace_path(namespace_path=None):
+    """Check a namespace path locally and return it as a tuple.
+
+    ``None`` and ``[]`` both mean the root namespace. Segments follow the same
+    rule as Secret names: a binding carries the path and the name as separate
+    fields, so neither is ever parsed out of the other.
+    """
+    if namespace_path is None:
+        return ()
+    if isinstance(namespace_path, str):
+        raise TypeError(
+            "namespace_path must be a list of segments, not a string; "
+            f"did you mean [{namespace_path!r}]?"
+        )
+    segments = tuple(namespace_path)
+    for segment in segments:
+        if not isinstance(segment, str):
+            raise TypeError(
+                f"namespace path segment must be a string, not {type(segment).__name__}"
+            )
+        if not _SECRET_NAME.fullmatch(segment):
+            raise ValueError(f"invalid namespace path segment: {segment!r}")
+    return segments
+
+
 def validate_env_variable(name: str) -> str:
     """Check an environment variable name locally and return it unchanged."""
     if not isinstance(name, str):
@@ -76,11 +101,12 @@ class EnvVarSecret:
     ('openai-prod', 'OPENAI_API_KEY')
     """
 
-    __slots__ = ("_secret", "_env_variable")
+    __slots__ = ("_secret", "_env_variable", "_namespace_path")
 
-    def __init__(self, secret: str, env_variable: str):
+    def __init__(self, secret: str, env_variable: str, *, namespace_path=None):
         self._secret = validate_secret_name(secret)
         self._env_variable = validate_env_variable(env_variable)
+        self._namespace_path = validate_namespace_path(namespace_path)
 
     @property
     def secret(self) -> str:
@@ -92,10 +118,20 @@ class EnvVarSecret:
         """The environment variable the value is delivered in."""
         return self._env_variable
 
+    @property
+    def namespace_path(self):
+        """The namespace path the Secret is addressed within, root when empty."""
+        return list(self._namespace_path)
+
     def __repr__(self) -> str:
+        path = (
+            f", namespace_path={list(self._namespace_path)!r}"
+            if self._namespace_path
+            else ""
+        )
         return (
             f"EnvVarSecret(secret={self._secret!r}, "
-            f"env_variable={self._env_variable!r})"
+            f"env_variable={self._env_variable!r}{path})"
         )
 
     def __eq__(self, other: object) -> bool:
@@ -103,10 +139,13 @@ class EnvVarSecret:
             isinstance(other, EnvVarSecret)
             and other._secret == self._secret
             and other._env_variable == self._env_variable
+            and other._namespace_path == self._namespace_path
         )
 
     def __hash__(self) -> int:
-        return hash((EnvVarSecret, self._secret, self._env_variable))
+        return hash(
+            (EnvVarSecret, self._secret, self._env_variable, self._namespace_path)
+        )
 
 
 class SecretInfo:

python/python/tests/test_first_class_function_slice1.py

@@ -14,6 +14,7 @@ from lancedb.functions import (
     FunctionVersion,
     PythonRuntimeSpec,
     RefreshColumnResult,
+    SecretReference,
 )
 from lancedb.table import AsyncTable
 
@@ -110,7 +111,9 @@ def test_function_version_identity_is_immutable_and_exact():
     version = FunctionVersion.from_json(json.dumps(value))
     assert version.name == "embed"
     assert version.version == "fv_01K3EXACT"
-    assert dict(version.secret_env_bindings) == {"HF_TOKEN": "hf-prod"}
+    assert dict(version.secret_env_bindings) == {
+        "HF_TOKEN": SecretReference(name="hf-prod")
+    }
 
     with pytest.raises((TypeError, ValueError)):
         version.version = "fv_changed"
@@ -299,7 +302,7 @@ def test_canonical_client_values_carry_bindings_and_no_credentials():
         json.dumps(job_result("remote_function_job.json"))
     )
     canonical = json.loads(version.to_canonical_json())
-    assert canonical["secret_env_bindings"] == {"HF_TOKEN": "hf-prod"}
+    assert canonical["secret_env_bindings"] == {"HF_TOKEN": {"name": "hf-prod"}}
     assert_no_secret_values(canonical)
 
 

python/python/tests/test_first_class_function_slice2.py

@@ -25,6 +25,7 @@ import lancedb
 from lancedb.background_loop import LOOP
 from lancedb.functions import (
     PythonRuntimeSpec,
+    SecretReference,
     UdfDefinition,
     _canonical_arrow_type,
     _GRAMMAR_PRIMITIVES,
@@ -94,6 +95,54 @@ def test_secret_bound_udf_matches_its_shared_registration_golden():
     )
 
 
+def test_a_namespaced_binding_records_the_path_and_the_name():
+    """A binding names the parts, so nothing has to be parsed back out.
+
+    A root binding carries no path at all, which is what keeps its wire shape
+    identical to one written before namespaces existed.
+    """
+    root = EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
+    assert root.namespace_path == []
+
+    nested = EnvVarSecret(
+        secret="openai-prod",
+        env_variable="OPENAI_API_KEY",
+        namespace_path=["prod", "vision"],
+    )
+    assert nested.namespace_path == ["prod", "vision"]
+    assert nested != root
+
+    bound = analyze_caption.bind_secrets([nested])
+    assert bound.secret_env_bindings == {
+        "OPENAI_API_KEY": SecretReference(
+            name="openai-prod", namespace_path=("prod", "vision")
+        )
+    }
+
+    at_root = analyze_caption.bind_secrets([root])
+    assert at_root.secret_env_bindings == {
+        "OPENAI_API_KEY": SecretReference(name="openai-prod")
+    }
+    canonical = json.loads(at_root.to_canonical_json())
+    assert canonical["secret_env_bindings"] == {
+        "OPENAI_API_KEY": {"name": "openai-prod"}
+    }
+
+
+def test_a_namespace_path_is_validated_locally():
+    # The charset is the service's, not a delimiter's: a reference is never
+    # joined, so a segment cannot make anything parse two ways.
+    with pytest.raises(ValueError):
+        EnvVarSecret(
+            secret="openai-prod", env_variable="K", namespace_path=["with$delim"]
+        )
+    with pytest.raises(ValueError):
+        EnvVarSecret(secret="openai-prod", env_variable="K", namespace_path=["a/b"])
+    # A bare string is a plausible mistake with the wrong meaning.
+    with pytest.raises(TypeError):
+        EnvVarSecret(secret="openai-prod", env_variable="K", namespace_path="prod")
+
+
 def test_an_unbound_request_carries_no_binding_at_all():
     """Binding is a registration-time decision, so the definition holds none.
 
@@ -167,7 +216,7 @@ def test_a_function_binds_at_most_sixteen_secrets():
 
         envelope = json.loads(normalize_score.registration_request.to_canonical_json())
         envelope["secret_env_bindings"] = {
-            f"TOKEN_{index}": f"secret-{index}" for index in range(17)
+            f"TOKEN_{index}": {"name": f"secret-{index}"} for index in range(17)
         }
 
         async def submit_envelope():
@@ -194,8 +243,8 @@ def test_binding_names_are_validated_below_the_python_api():
         )
         envelope = json.loads(analyze_caption.registration_request.to_canonical_json())
         envelope["secret_env_bindings"] = {
-            "BAD=NAME": "openai-prod",
-            "TOKEN_0": "secret-0",
+            "BAD=NAME": {"name": "openai-prod"},
+            "TOKEN_0": {"name": "secret-0"},
         }
         envelope["runtime"]["env"]["TOKEN_0"] = "public"
 
@@ -1573,10 +1622,14 @@ def test_remote_registration_sends_bindings_and_never_a_credential():
             secrets=[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")],
         )
 
-    assert dict(created.secret_env_bindings) == {"OPENAI_API_KEY": "openai-prod"}
+    assert dict(created.secret_env_bindings) == {
+        "OPENAI_API_KEY": SecretReference(name="openai-prod")
+    }
     path, create_request = state["requests"][0]
     assert path == "/v1/functions/create"
-    assert create_request["secret_env_bindings"] == {"OPENAI_API_KEY": "openai-prod"}
+    assert create_request["secret_env_bindings"] == {
+        "OPENAI_API_KEY": {"name": "openai-prod"}
+    }
     # The request names a Secret and carries nothing that could be one.
     assert create_request == json.loads(
         analyze_caption.bind_secrets(

python/src/connection.rs

@@ -708,10 +708,14 @@ impl Connection {
         self_: PyRef<'_, Self>,
         name: String,
         value: String,
+        namespace_path: Vec<String>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
-            inner.create_secret(name, value).await.infer_error()
+            inner
+                .create_secret(name, value, &namespace_path)
+                .await
+                .infer_error()
         })
     }
 
@@ -719,33 +723,51 @@ impl Connection {
         self_: PyRef<'_, Self>,
         name: String,
         value: String,
+        namespace_path: Vec<String>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
-            inner.alter_secret(name, value).await.infer_error()
+            inner
+                .alter_secret(name, value, &namespace_path)
+                .await
+                .infer_error()
         })
     }
 
-    pub fn list_secrets(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn list_secrets(
+        self_: PyRef<'_, Self>,
+        namespace_path: Vec<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
-            inner.list_secrets().await.infer_error()
+            inner.list_secrets(&namespace_path).await.infer_error()
         })
     }
 
-    pub fn drop_secret(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
+    pub fn drop_secret(
+        self_: PyRef<'_, Self>,
+        name: String,
+        namespace_path: Vec<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
-            inner.drop_secret(name).await.infer_error()
+            inner.drop_secret(name, &namespace_path).await.infer_error()
         })
     }
 
     /// Name and timestamps as a plain mapping. `SecretInfo` carries no value,
     /// so there is none to filter out here.
-    pub fn describe_secret(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
+    pub fn describe_secret(
+        self_: PyRef<'_, Self>,
+        name: String,
+        namespace_path: Vec<String>,
+    ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
-            let info = inner.describe_secret(name).await.infer_error()?;
+            let info = inner
+                .describe_secret(name, &namespace_path)
+                .await
+                .infer_error()?;
             Ok(HashMap::from([
                 ("name".to_string(), info.name),
                 ("created_at".to_string(), info.created_at),