diff --git a/docs/src/js/interfaces/ClientConfig.md b/docs/src/js/interfaces/ClientConfig.md index 94cee5b5e..2b4984b8d 100644 --- a/docs/src/js/interfaces/ClientConfig.md +++ b/docs/src/js/interfaces/ClientConfig.md @@ -22,6 +22,10 @@ optional extraHeaders: Record; optional idDelimiter: string; ``` +The delimiter joining a namespace path and a name into one object +identifier. `"$"` is the only supported value, and leaving this unset is +how to get it; anything else is rejected when the connection is created. + *** ### retryConfig? diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 96161fbac..80899b964 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -136,6 +136,10 @@ listing a storage directory. ::: lancedb.functions.UdfDefinition +::: lancedb.secrets.EnvVarSecret + +::: lancedb.secrets.SecretInfo + ::: lancedb.functions.FunctionRegistrationRequest ::: lancedb.functions.FunctionArtifactRequest diff --git a/nodejs/__test__/catalog.test.ts b/nodejs/__test__/catalog.test.ts index d44103ab9..f9158c94f 100644 --- a/nodejs/__test__/catalog.test.ts +++ b/nodejs/__test__/catalog.test.ts @@ -87,7 +87,7 @@ describe("remote catalog", () => { expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create"); expect(requests[0].body).toEqual({ mode: "ExistOk" }); expect(requests[5].url).toBe( - "/v1/namespace/%24/list?limit=1&page_token=a%2Fb", + "/v1/namespace/$/list?limit=1&page_token=a%2Fb", ); expect(requests[6].body).toEqual({ mode: "Skip", diff --git a/nodejs/src/remote.rs b/nodejs/src/remote.rs index 784631267..c2d671735 100644 --- a/nodejs/src/remote.rs +++ b/nodejs/src/remote.rs @@ -93,6 +93,9 @@ pub struct ClientConfig { pub retry_config: Option, pub timeout_config: Option, pub extra_headers: Option>, + /// The delimiter joining a namespace path and a name into one object + /// identifier. `"$"` is the only supported value, and leaving this unset is + /// how to get it; anything else is rejected when the connection is created. pub id_delimiter: Option, pub tls_config: Option, /// User identifier for tracking purposes. diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 19114e1a1..05ad664e1 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -37,6 +37,8 @@ from .functions import ( UdfDefinition as UdfDefinition, udf as udf, ) +from .secrets import EnvVarSecret as EnvVarSecret +from .secrets import SecretInfo as SecretInfo from .materialized_view import ( AsyncMaterializedView, MaterializedView, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index f267b3a55..a0039df05 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -153,6 +153,21 @@ class Connection(object): async def get_function(self, name: str, version: str) -> str: ... async def list_functions(self) -> List[str]: ... async def drop_function(self, name: str, version: str) -> bool: ... + async def create_secret( + self, name: str, value: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def alter_secret( + self, name: str, value: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def list_secrets( + self, namespace_path: Optional[List[str]] = None + ) -> List[str]: ... + async def drop_secret( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def describe_secret( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Tuple[str, int, int]: ... async def list_jobs(self) -> List[JobInfo]: ... async def cancel_job(self, job_id: str) -> bool: ... async def execute_query_async( diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 4a8820cef..d17e897cf 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -17,6 +17,7 @@ from typing import ( List, Literal, Optional, + Sequence, Union, ) from uuid import UUID @@ -57,6 +58,12 @@ from .materialized_view import ( SelectArg, normalize_select, ) +from .secrets import ( + EnvVarSecret, + SecretInfo, + validate_namespace_path, + validate_secret_name, +) from .table import ( AsyncTable, LanceTable, @@ -742,16 +749,50 @@ class DBConnection(EnforceOverrides): """ raise NotImplementedError("serialize is not supported for this connection type") - def create_function(self, definition: UdfDefinition) -> FunctionVersion: + def create_function( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> FunctionVersion: """Build and register a scalar Python UDF, then return its version. The server builds the OCI image and registers the completed artifact. This is the blocking counterpart of :meth:`create_function_async`. Local connections raise ``NotImplementedError``. - """ - return self.create_function_async(definition).wait() - def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: + Parameters + ---------- + definition : UdfDefinition + A callable decorated with [udf][lancedb.udf]. + secrets : sequence of EnvVarSecret, optional + One [EnvVarSecret][lancedb.secrets.EnvVarSecret] per credential the + Function needs, each naming a Secret and the environment variable + its value arrives in. The Function's source is unchanged by this; + it reads the variable the way it already did. + + Examples + -------- + ```python + db.create_secret("openai-prod", os.environ["OPENAI_API_KEY"]) + db.create_function( + analyze_caption, + secrets=[ + EnvVarSecret( + secret_name="openai-prod", env_variable="OPENAI_API_KEY" + ) + ], + ) + ``` + """ + return self.create_function_async(definition, secrets=secrets).wait() + + def create_function_async( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> Job[FunctionVersion]: """Submit a scalar Python UDF for building and registration. The server-side job builds the OCI image, then registers the completed @@ -798,6 +839,70 @@ class DBConnection(EnforceOverrides): "Function catalog operations are not supported for this connection type" ) + def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Create a named Secret in this database. + + Fails if the name is taken, so a create never silently becomes a + rotation. Nothing reads the value back: it is bound to a Function by + name and resolved by the service when that Function runs. Local + connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Replace the credential behind an existing Secret. + + Fails if it does not exist. Every Function bound to the Secret uses the + new value from its next job, and no new Function version is created -- + which is how a rotation reaches columns pinned to a version registered + before it. Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + """The names of every Secret in this database. + + Names only. No method returns a stored credential, by construction + rather than by policy. Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a Secret. + + Functions bound to it fail at their next job, naming the Secret; that + is the revocation path. The name becomes free to reuse, and a new + Secret under it is picked up by everything still bound to that name. + Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + """What this database records about a Secret: name and timestamps. + + Never the value -- there is no code path that could return one. Local + connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + def open_job(self, job_id: str) -> Job: """Open a server-side job by id, returning a handle with its record already populated. @@ -1557,8 +1662,13 @@ class LanceDBConnection(DBConnection): return Job(LOOP.run(self._conn.open_job(job_id))) @override - def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: - job = LOOP.run(self._conn.create_function_async(definition)) + def create_function_async( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> Job[FunctionVersion]: + job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets)) return Job(job) @override @@ -1573,6 +1683,34 @@ class LanceDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path)) + + @override + def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path)) + + @override + def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path)) + + @override + def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path)) + + @override + def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path)) + @override def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" @@ -2422,19 +2560,24 @@ class AsyncConnection(object): return AsyncJob(await self._inner.open_job(job_id)) async def create_function_async( - self, definition: UdfDefinition + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, ) -> AsyncJob[FunctionVersion]: """Submit a scalar Python UDF for building and registration. The server-side job builds the OCI image, then registers the completed artifact. Waiting on the job returns the immutable Function version. - Local connections raise ``NotImplementedError``. + ``secrets`` is a sequence of + [EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and + the environment variable its value arrives in. Local connections raise + ``NotImplementedError``. """ if not isinstance(definition, UdfDefinition): raise TypeError("create_function_async requires a @udf definition") - inner = await self._inner.create_function_async( - definition.registration_request.to_canonical_json() - ) + request = definition.bind_secrets(secrets) + inner = await self._inner.create_function_async(request.to_canonical_json()) return _typed_job(inner, FunctionVersion.from_json) async def get_function(self, name: str, *, version: str) -> FunctionVersion: @@ -2456,6 +2599,64 @@ class AsyncConnection(object): """Remove the current name binding, retaining the object and its history.""" return await self._inner.drop_function(name, version) + async def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Create a named Secret in this database. + + Fails if the name is taken, so a create never silently becomes a + rotation. Nothing reads the value back. + """ + await self._inner.create_secret( + validate_secret_name(name), + value, + list(validate_namespace_path(namespace_path)), + ) + + async def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Replace the credential behind an existing Secret. + + Fails if it does not exist. Bound Functions use the new value from + their next job, with no new Function version. + """ + await self._inner.alter_secret( + validate_secret_name(name), + value, + list(validate_namespace_path(namespace_path)), + ) + + async def list_secrets( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[str]: + """The names of every Secret in this database. Names only.""" + return await self._inner.list_secrets( + list(validate_namespace_path(namespace_path)) + ) + + async def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a Secret. Bound Functions fail at their next job.""" + await self._inner.drop_secret( + validate_secret_name(name), list(validate_namespace_path(namespace_path)) + ) + + async def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + """What this database records about a Secret. Never the value.""" + name, created_at_millis, updated_at_millis = await self._inner.describe_secret( + validate_secret_name(name), + list(validate_namespace_path(namespace_path)), + ) + return SecretInfo( + name=name, + created_at_millis=created_at_millis, + updated_at_millis=updated_at_millis, + ) + async def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" return await self._inner.list_jobs() diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 692bc5756..8c6864cc3 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -4,7 +4,7 @@ """Canonical Function values exchanged with LanceDB Enterprise services. These immutable models contain client/wire state only. Catalog persistence, -environment bake, and execution are owned by Sophon. +environment bake, secret resolution, and execution are owned by Sophon. ``RefreshColumnResult`` is also the backend-neutral result of a local expression-backed refresh job. """ @@ -25,7 +25,7 @@ import re import sys import textwrap import types -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import date, datetime from typing import ( Annotated, @@ -51,6 +51,7 @@ from pydantic import ( ) from .schema import is_blob_v2_field as _is_blob_v2_field +from .secrets import EnvVarSecret _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1) _UInt32 = conint(strict=True, ge=0, le=2**32 - 1) @@ -227,6 +228,33 @@ class FunctionOutput(_OpenRemoteValue): fields: tuple[FunctionResultField, ...] = () +class SecretReference(_RemoteValue): + """Where a Secret lives, carried as its parts rather than as one string. + + A joined id would need a delimiter, and a delimiter has to be excluded from + every name and segment forever, agreed on by both sides, and re-agreed each + time either grows a new way to be configured. Naming the parts settles all + of that: nothing here is parsed, so nothing can parse two ways. + """ + + name: str + namespace_path: tuple[str, ...] = () + + +class SecretBinding(_RemoteValue): + """How a Secret reaches the Function that binds it. + + One list rather than a field per delivery mode: a binding is the concept, + and how it arrives is a property of one. ``kind`` is open, so a binding a + newer service introduces decodes here instead of failing the whole + FunctionVersion. + """ + + kind: str + variable: Optional[str] = None + secret_ref: Optional[SecretReference] = None + + class FunctionSignature(_RemoteValue): inputs: tuple[FunctionParameter, ...] output: FunctionOutput @@ -326,6 +354,7 @@ class FunctionVersion(_RemoteValue): version: _ObjectVersion image: FunctionImage signature: FunctionSignature + secret_bindings: tuple[SecretBinding, ...] = () created_at: str metadata: Mapping[str, str] disabled: bool @@ -395,12 +424,18 @@ class FunctionVersion(_RemoteValue): class FunctionRegistrationRequest(_RemoteValue): - """Stable remote registration envelope produced by :func:`udf`.""" + """Stable remote registration envelope produced by :func:`udf`. + + Credential values deliberately have no field here. The only secret-shaped + thing a client sends is ``secret_bindings``: the name of a Secret the + database already holds, which the remote service resolves at execution. + """ name: str artifact: FunctionArtifactRequest signature: FunctionSignature runtime: PythonRuntimeSpec + secret_bindings: tuple[SecretBinding, ...] = () class FunctionVersionRef(_OpenRemoteValue): @@ -552,6 +587,7 @@ class RefreshColumnResult(_RemoteValue): _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") + _FUNCTION_BLOB_V2_TYPE = "blob_v2" _ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name" _BLOB_V2_EXTENSION_NAME = "lance.blob.v2" @@ -1293,9 +1329,73 @@ class UdfDefinition: @property def registration_request(self) -> FunctionRegistrationRequest: - """The immutable request sent by ``create_function_async``.""" + """The immutable request sent by ``create_function_async``. + + Carries no secret bindings. Binding is a registration-time decision, + so a Function bound to Secrets is registered through :meth:`bind_secrets`, + which is what ``create_function`` calls. + """ return self._request + def bind_secrets( + self, secrets: Optional[Sequence[EnvVarSecret]] + ) -> FunctionRegistrationRequest: + """The registration request for this definition bound to ``secrets``. + + Binding does not change the Function's source: each + [EnvVarSecret][lancedb.secrets.EnvVarSecret] names a Secret and the + environment variable its value should arrive in, and the Function reads + that variable the way it already did. Whether the named Secrets exist is + the server's answer, not this one. + """ + bindings = () if secrets is None else tuple(secrets) + wrong_type = [ + binding for binding in bindings if not isinstance(binding, EnvVarSecret) + ] + if wrong_type: + kinds = sorted({type(binding).__name__ for binding in wrong_type}) + raise TypeError( + f"Function secrets must be EnvVarSecret values, not {kinds!r}; a " + "credential value is never sent to this API" + ) + variables = [binding.env_variable for binding in bindings] + duplicates = sorted({name for name in variables if variables.count(name) > 1}) + if duplicates: + raise ValueError( + "a Function binds each environment variable once; duplicated: " + f"{duplicates!r}" + ) + # `env` is ordinary configuration carried in the definition, so a name in + # both would have a value visible in the Function's record and a value + # that is not. Refuse rather than pick. + environment = self._request.runtime.env or {} + overlap = sorted(set(environment) & set(variables)) + if overlap: + raise ValueError( + f"Function env and secret bindings must be disjoint: {overlap!r}" + ) + if not bindings: + return self._request + # Sorted, because the list is carried in the FunctionVersion hash and a + # caller's argument order is not part of what a Function is. + resolved = tuple( + sorted( + ( + SecretBinding( + kind="env", + variable=binding.env_variable, + secret_ref=SecretReference( + name=binding.secret_name, + namespace_path=tuple(binding.secret_namespace_path), + ), + ) + for binding in bindings + ), + key=lambda binding: (binding.kind, binding.variable or ""), + ) + ) + return self._request._copy(update={"secret_bindings": resolved}) + def __call__(self, *args, **kwargs): return self._function(*args, **kwargs) @@ -1360,7 +1460,9 @@ def udf( conda_channels : sequence of str, optional Conda channels in priority order; requires ``conda``. env : mapping of str to str, optional - Environment variables included in the Function definition. + Environment variables included in the Function definition. Not for + credentials -- these are ordinary configuration, stored with the + Function and visible wherever it is. python_version : str, optional Remote Python major/minor version. Defaults to the client version. gpu : bool, default False diff --git a/python/python/lancedb/remote/__init__.py b/python/python/lancedb/remote/__init__.py index b70a869a3..dbcaa1b64 100644 --- a/python/python/lancedb/remote/__init__.py +++ b/python/python/lancedb/remote/__init__.py @@ -173,7 +173,10 @@ class ClientConfig: extra_headers: Optional[dict] Additional headers to include in requests. id_delimiter: Optional[str] - The delimiter to use when constructing object identifiers. + The delimiter joining a namespace path and a name into one object + identifier. ``"$"`` is the only supported value, and leaving this + unset is how to get it; anything else is rejected when the connection + is created. tls_config: Optional[TlsConfig] TLS/mTLS configuration for secure connections. header_provider: Optional[HeaderProvider] diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index aceebf987..c9e857495 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -8,7 +8,16 @@ import json import logging from concurrent.futures import ThreadPoolExecutor import sys -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Sequence, + Union, +) from urllib.parse import urlparse from uuid import UUID import warnings @@ -30,6 +39,7 @@ from ..job import AsyncJob, Job from ..sql import Query as SqlQuery from ..sql import QueryDescription from ..materialized_view import MaterializedView, SelectArg +from ..secrets import EnvVarSecret, SecretInfo if TYPE_CHECKING: from .._lancedb import JobInfo @@ -845,8 +855,14 @@ class RemoteDBConnection(DBConnection): return Job(LOOP.run(self._conn.open_job(job_id))) @override - def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: - return Job(LOOP.run(self._conn.create_function_async(definition))) + def create_function_async( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> Job[FunctionVersion]: + job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets)) + return Job(job) @override def get_function(self, name: str, *, version: str) -> FunctionVersion: @@ -860,6 +876,34 @@ class RemoteDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path)) + + @override + def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path)) + + @override + def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path)) + + @override + def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path)) + + @override + def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path)) + @override def list_jobs(self) -> List["JobInfo"]: """List server-side jobs across the database's tables.""" diff --git a/python/python/lancedb/secrets.py b/python/python/lancedb/secrets.py new file mode 100644 index 000000000..75ac9e340 --- /dev/null +++ b/python/python/lancedb/secrets.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Named Secrets, and the bindings that deliver them to Functions. + +A Secret is a database-scoped named credential. Nothing in this module holds a +value: :class:`EnvVarSecret` names one and says which environment variable it +should arrive in, and the value is resolved by the remote service when a +Function bound to it runs. No API returns a stored credential, by construction +rather than by policy -- there is no code path that could. +""" + +from __future__ import annotations + +import re + +# The same characters LanceDB already admits in a namespace or table name, and +# no positional rule on top of them: a segment may begin with `_`, `-` or `.` +# today, so anything narrower would put Secrets out of reach inside namespaces +# that already exist. Matches the service, which admits the same set. +_SECRET_NAME = re.compile(r"^[A-Za-z0-9_.-]{1,255}$") +_ENV_VARIABLE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def validate_secret_name(name: str) -> str: + """Check a Secret name locally and return it unchanged.""" + if not isinstance(name, str): + raise TypeError(f"Secret name must be a string, not {type(name).__name__}") + if not _SECRET_NAME.fullmatch(name): + raise ValueError(f"invalid Secret name: {name!r}") + return name + + +def validate_namespace_path(namespace_path=None): + """Check a namespace path locally and return it as a tuple. + + ``None`` and ``[]`` both mean the root namespace. Segments follow the same + rule as Secret names: a binding carries the path and the name as separate + fields, so neither is ever parsed out of the other. + """ + if namespace_path is None: + return () + if isinstance(namespace_path, str): + raise TypeError( + "namespace_path must be a list of segments, not a string; " + f"did you mean [{namespace_path!r}]?" + ) + segments = tuple(namespace_path) + for segment in segments: + if not isinstance(segment, str): + raise TypeError( + f"namespace path segment must be a string, not {type(segment).__name__}" + ) + if not _SECRET_NAME.fullmatch(segment): + raise ValueError(f"invalid namespace path segment: {segment!r}") + return segments + + +def validate_env_variable(name: str) -> str: + """Check an environment variable name locally and return it unchanged.""" + if not isinstance(name, str): + raise TypeError( + f"environment variable name must be a string, not {type(name).__name__}" + ) + if not _ENV_VARIABLE.fullmatch(name): + raise ValueError(f"invalid environment variable name: {name!r}") + return name + + +class EnvVarSecret: + """A Secret bound to the environment variable a Function's library reads. + + Pass these in the ``secrets`` sequence of + [DBConnection.create_function][lancedb.db.DBConnection.create_function]. The + Function's source is unchanged by binding: it reads ``OPENAI_API_KEY`` the + way it always did, and the binding is what puts a value there. + + This is a local value. Constructing it contacts no server, so it always + succeeds and says nothing about whether the Secret exists; that is checked + at registration, where a mistyped Secret name surfaces as a clear "does not + exist" naming both the Secret and the variable bound to it. A mistyped + *variable* name cannot be caught anywhere -- nothing knows which variables a + Function reads -- so it surfaces on the first rows instead. + + The type exists so a credential cannot be passed by accident. A bare string + in the same position is a plausible-looking mistake with the opposite + meaning, and it reads identically in a diff. + + Parameters + ---------- + secret_name : str + The Secret's database-scoped name. + env_variable : str + The environment variable the Function reads it from. + secret_namespace_path : list of str, optional + The namespace the Secret is addressed within. ``None`` and ``[]`` both + mean the root namespace. Carried beside the name rather than joined + into it, so neither is ever parsed back out of the other. + + Examples + -------- + >>> from lancedb import EnvVarSecret + >>> binding = EnvVarSecret( + ... secret_name="openai-prod", env_variable="OPENAI_API_KEY" + ... ) + >>> binding.secret_name, binding.env_variable + ('openai-prod', 'OPENAI_API_KEY') + """ + + __slots__ = ("_secret_name", "_env_variable", "_secret_namespace_path") + + def __init__( + self, secret_name: str, env_variable: str, *, secret_namespace_path=None + ): + self._secret_name = validate_secret_name(secret_name) + self._env_variable = validate_env_variable(env_variable) + self._secret_namespace_path = validate_namespace_path(secret_namespace_path) + + @property + def secret_name(self) -> str: + """The Secret's database-scoped name.""" + return self._secret_name + + @property + def env_variable(self) -> str: + """The environment variable the value is delivered in.""" + return self._env_variable + + @property + def secret_namespace_path(self): + """The namespace path the Secret is addressed within, root when empty.""" + return list(self._secret_namespace_path) + + def __repr__(self) -> str: + path = ( + f", secret_namespace_path={list(self._secret_namespace_path)!r}" + if self._secret_namespace_path + else "" + ) + return ( + f"EnvVarSecret(secret_name={self._secret_name!r}, " + f"env_variable={self._env_variable!r}{path})" + ) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, EnvVarSecret) + and other._secret_name == self._secret_name + and other._env_variable == self._env_variable + and other._secret_namespace_path == self._secret_namespace_path + ) + + def __hash__(self) -> int: + return hash( + ( + EnvVarSecret, + self._secret_name, + self._env_variable, + self._secret_namespace_path, + ) + ) + + +class SecretInfo: + """What a database records about a Secret. Never its value. + + Returned by + [DBConnection.describe_secret][lancedb.db.DBConnection.describe_secret]. + """ + + __slots__ = ("_name", "_created_at_millis", "_updated_at_millis") + + def __init__(self, name: str, created_at_millis: int, updated_at_millis: int): + self._name = name + self._created_at_millis = created_at_millis + self._updated_at_millis = updated_at_millis + + @property + def name(self) -> str: + """The Secret's database-scoped name.""" + return self._name + + @property + def created_at_millis(self) -> int: + """When the Secret was created, in milliseconds since the Unix epoch.""" + return self._created_at_millis + + @property + def updated_at_millis(self) -> int: + """When the Secret's value was last rotated, in epoch milliseconds. + + The only observable that a rotation landed: no API returns a credential, + so a caller confirms ``alter_secret`` took effect by watching this move. + """ + return self._updated_at_millis + + @classmethod + def from_json(cls, value: dict) -> "SecretInfo": + return cls( + name=value["name"], + created_at_millis=value["created_at_millis"], + updated_at_millis=value["updated_at_millis"], + ) + + def __repr__(self) -> str: + return ( + f"SecretInfo(name={self._name!r}, " + f"created_at_millis={self._created_at_millis!r}, " + f"updated_at_millis={self._updated_at_millis!r})" + ) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, SecretInfo) + and other._name == self._name + and other._created_at_millis == self._created_at_millis + and other._updated_at_millis == self._updated_at_millis + ) + + +__all__ = [ + "EnvVarSecret", + "SecretInfo", + "validate_env_variable", + "validate_secret_name", +] diff --git a/python/python/tests/test_catalog.py b/python/python/tests/test_catalog.py index 63386a998..ca5d68fdf 100644 --- a/python/python/tests/test_catalog.py +++ b/python/python/tests/test_catalog.py @@ -91,7 +91,7 @@ def test_catalog_sync_scope_and_serialization(catalog_server): assert requests[0][0] == "/v1/namespace/team%2Fsearch/create" assert requests[0][2] == {"mode": "ExistOk"} assert requests[1][0] == "/v1/namespace/team%2Fsearch/describe" - assert requests[4][0] == "/v1/namespace/%24/list?limit=1&page_token=a%2Fb" + assert requests[4][0] == "/v1/namespace/$/list?limit=1&page_token=a%2Fb" assert requests[5][2] == {"mode": "Skip", "behavior": "Restrict"} for i, (_, headers, _) in enumerate(requests): headers = {key.lower(): value for key, value in headers.items()} diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index 459212b0f..fbcfa03b1 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -13,7 +13,9 @@ from lancedb.functions import ( FunctionBinding, FunctionVersion, PythonRuntimeSpec, + SecretBinding, RefreshColumnResult, + SecretReference, ) from lancedb.table import AsyncTable @@ -37,6 +39,22 @@ def job_result(name: str) -> dict: return json.loads(fixture(name))["result"] +def assert_no_secret_values(value): + """No client value models a resolved credential, at any nesting depth.""" + if isinstance(value, dict): + for key, child in value.items(): + assert key not in { + "secret_value", + "secret_values", + "resolved_secret", + "resolved_secrets", + } + assert_no_secret_values(child) + elif isinstance(value, list): + for child in value: + assert_no_secret_values(child) + + def test_public_function_values_are_in_api_reference(): docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md" rendered = docs.read_text() @@ -96,6 +114,11 @@ def test_function_version_identity_is_immutable_and_exact(): assert version.version == "1" assert version.image.manifest_digest.startswith("sha256:") assert version.version != version.image.manifest_digest + assert list(version.secret_bindings) == [ + SecretBinding( + kind="env", variable="HF_TOKEN", secret_ref=SecretReference(name="hf-prod") + ) + ] with pytest.raises((TypeError, ValueError)): version.version = "1" @@ -282,6 +305,31 @@ def test_refresh_result_rejects_non_u64_values(field): RefreshColumnResult.from_json(json.dumps(value)) +def test_canonical_client_values_carry_bindings_and_no_credentials(): + """A binding names a Secret; the credential behind it has no client field.""" + version = FunctionVersion.from_json( + json.dumps(job_result("remote_function_job.json")) + ) + canonical = json.loads(version.to_canonical_json()) + assert canonical["secret_bindings"] == [ + {"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}} + ] + assert_no_secret_values(canonical) + + +def test_a_version_without_bindings_omits_the_field_in_both_directions(): + """A Function that binds nothing carries no ``secret_bindings`` key. + + Absent decodes as an empty list, and an empty list serializes back to + absent. + """ + value = job_result("remote_function_job.json") + del value["secret_bindings"] + version = FunctionVersion.from_json(json.dumps(value)) + assert list(version.secret_bindings) == [] + assert "secret_bindings" not in json.loads(version.to_canonical_json()) + + class _FunctionDeclarationInner: def __init__(self): self.calls = [] diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 8d4f75f67..1cd8d4cf5 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -11,6 +11,7 @@ import types from datetime import date import http.server import json +import os from pathlib import Path import subprocess import sys @@ -24,11 +25,14 @@ import pytest import lancedb from lancedb.functions import ( PythonRuntimeSpec, + SecretBinding, + SecretReference, UdfDefinition, _canonical_arrow_type, _GRAMMAR_PRIMITIVES, udf, ) +from lancedb.secrets import EnvVarSecret THRESHOLD = 20 _CACHE = None @@ -59,6 +63,15 @@ def normalize_score(value: float) -> float: return value / 100.0 +@udf( + pip=["openai==3.7.0"], + env={"MODE": "test"}, + python_version="3.12", +) +def analyze_caption(caption: str) -> str: + return caption.strip() + + def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): assert isinstance(normalize_score, UdfDefinition) assert normalize_score(25.0) == 0.25 @@ -75,6 +88,286 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): } +def test_secret_bound_udf_matches_its_shared_registration_golden(): + assert analyze_caption(" hello ") == "hello" + bound = analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")] + ) + assert ( + bound.to_canonical_json() + == (FIXTURES / "remote_function_secret_registration_request.canonical.json") + .read_text() + .strip() + ) + + +def test_a_namespaced_binding_records_the_path_and_the_name(): + """A binding names the parts, so nothing has to be parsed back out. + + A root binding carries no path at all: the field is absent rather than an + empty list, so a binding states a namespace only when it has one. + """ + root = EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") + assert root.secret_namespace_path == [] + + nested = EnvVarSecret( + secret_name="openai-prod", + env_variable="OPENAI_API_KEY", + secret_namespace_path=["prod", "vision"], + ) + assert nested.secret_namespace_path == ["prod", "vision"] + assert nested != root + + bound = analyze_caption.bind_secrets([nested]) + assert list(bound.secret_bindings) == [ + SecretBinding( + kind="env", + variable="OPENAI_API_KEY", + secret_ref=SecretReference( + name="openai-prod", namespace_path=("prod", "vision") + ), + ) + ] + + at_root = analyze_caption.bind_secrets([root]) + assert list(at_root.secret_bindings) == [ + SecretBinding( + kind="env", + variable="OPENAI_API_KEY", + secret_ref=SecretReference(name="openai-prod"), + ) + ] + # A root binding carries no path at all on the wire. + canonical = json.loads(at_root.to_canonical_json()) + assert canonical["secret_bindings"] == [ + { + "kind": "env", + "variable": "OPENAI_API_KEY", + "secret_ref": {"name": "openai-prod"}, + } + ] + + +def test_a_namespace_path_is_validated_locally(): + # The charset is the service's, not a delimiter's: a reference is never + # joined, so a segment cannot make anything parse two ways. + with pytest.raises(ValueError): + EnvVarSecret( + secret_name="openai-prod", + env_variable="K", + secret_namespace_path=["with$delim"], + ) + with pytest.raises(ValueError): + EnvVarSecret( + secret_name="openai-prod", env_variable="K", secret_namespace_path=["a/b"] + ) + # A bare string is a plausible mistake with the wrong meaning. + with pytest.raises(TypeError): + EnvVarSecret( + secret_name="openai-prod", env_variable="K", secret_namespace_path="prod" + ) + + +def test_an_unbound_request_carries_no_binding_at_all(): + """Binding is a registration-time decision, so the definition holds none. + + The decorator declares nothing about secrets, which is what makes the PRD's + claim true: a Function's source and its registration request are identical + whether or not a credential is later bound to it. + """ + unbound = json.loads(analyze_caption.registration_request.to_canonical_json()) + assert "secret_bindings" not in unbound + assert "OPENAI_API_KEY" not in json.dumps(unbound) + + +def test_binding_a_secret_leaves_the_packaged_artifact_untouched(): + """The artifact is source bytes and nothing else, with or without secrets.""" + bound = analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")] + ) + assert bound.artifact == analyze_caption.registration_request.artifact + assert bound.artifact.digest == analyze_caption.registration_request.artifact.digest + + +def test_a_function_declaring_no_secret_is_registered_exactly_as_before(): + """The compatibility claim: nothing about the no-secret path moves.""" + assert ( + normalize_score.bind_secrets(None).to_canonical_json() + == normalize_score.registration_request.to_canonical_json() + ) + assert ( + "secret_bindings" + not in normalize_score.registration_request.to_canonical_json() + ) + + +def test_a_function_binds_each_variable_once(): + with pytest.raises(ValueError, match="binds each environment variable once"): + analyze_caption.bind_secrets( + [ + EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY"), + EnvVarSecret( + secret_name="openai-staging", env_variable="OPENAI_API_KEY" + ), + ] + ) + + +def test_bindings_may_not_collide_with_plain_configuration(): + """`env` is stored with the Function; a Secret is not. Refuse, do not pick.""" + with pytest.raises(ValueError, match="must be disjoint"): + analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="mode-prod", env_variable="MODE")] + ) + + +def test_a_binding_envelope_reaches_the_service_for_it_to_judge(): + """Binding rules are the service's: it owns the runtime the names land in. + + The client sends what it was given, so a rule it duplicated could disagree + with the service's without either side noticing. What is checked here is + that the envelope arrives intact -- the shape the service judges is the + shape the caller wrote. + """ + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + bindings = [ + EnvVarSecret(secret_name=f"secret-{index}", env_variable=f"TOKEN_{index}") + for index in range(17) + ] + db.create_function(normalize_score, secrets=bindings) + + sent = state["requests"][0][1] + assert len(sent["secret_bindings"]) == 17 + assert { + "kind": "env", + "variable": "TOKEN_0", + "secret_ref": {"name": "secret-0"}, + } in sent["secret_bindings"] + + +_SECRET_DEBUG_LOG_SOURCE = """ +import http.server +import json +import threading + +import lancedb + + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + payload = json.dumps({}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +server = http.server.ThreadingHTTPServer(("localhost", 0), Handler) +threading.Thread(target=server.serve_forever, daemon=True).start() +try: + db = lancedb.connect( + "db://dev", + api_key="API_KEY_SENTINEL", + host_override="http://localhost:%d" % server.server_address[1], + client_config={"retry_config": {"retries": 0}}, + ) + db.create_secret("openai-prod", "SECRET_VALUE_SENTINEL") +finally: + server.shutdown() +""" + + +def test_a_credential_never_reaches_a_debug_log(tmp_path): + """The logger sees the serialized body, so no value-side redaction reaches it. + + Runs in a subprocess because the Rust logger reads ``LANCEDB_LOG`` once, at + import. + """ + script = tmp_path / "write_secret.py" + script.write_text(_SECRET_DEBUG_LOG_SOURCE) + + result = subprocess.run( + [sys.executable, str(script)], + check=True, + capture_output=True, + text=True, + env={**os.environ, "LANCEDB_LOG": "debug"}, + ) + output = result.stdout + result.stderr + + # Without this the test passes when debug logging is simply off. + assert "Sending request_id=" in output, output + assert "SECRET_VALUE_SENTINEL" not in output + assert "API_KEY_SENTINEL" not in output + + +def test_a_credential_value_is_rejected_in_the_binding_position(): + """The one mistake the typed binding exists to stop.""" + with pytest.raises(TypeError, match="EnvVarSecret"): + analyze_caption.bind_secrets(["sk-live-0001"]) + + +@pytest.mark.parametrize( + ("secret", "variable", "message"), + [ + ("openai-prod", "not-a-var", "invalid environment variable name"), + ("openai-prod", "API-TOKEN", "invalid environment variable name"), + ("not a name", "API_TOKEN", "invalid Secret name"), + ("openai$prod", "API_TOKEN", "invalid Secret name"), + ], +) +def test_a_binding_validates_both_names_locally(secret, variable, message): + with pytest.raises(ValueError, match=message): + EnvVarSecret(secret_name=secret, env_variable=variable) + + +def test_a_secret_name_admits_what_a_namespace_name_does(): + """A Secret has to be nameable wherever a namespace already is. + + LanceDB namespace and table names are `[A-Za-z0-9_.-]` with no rule about + which character comes first, so a name may lead with `_`, `-` or `.`. + Anything narrower here would leave Secrets unaddressable inside namespaces + that already exist -- the reason periods are admitted is the reason the + edges are too. + """ + for name in ["openai.prod.v1", ".hidden", "_internal", "-lead", "trailing."]: + binding = EnvVarSecret(secret_name=name, env_variable="OPENAI_API_KEY") + assert binding.secret_name == name + + for name in ["", "with/slash", "with$delimiter", "a" * 256]: + with pytest.raises(ValueError, match="invalid Secret name"): + EnvVarSecret(secret_name=name, env_variable="OPENAI_API_KEY") + + # A namespace segment follows the same rule, and LanceDB already admits + # these shapes as namespace names -- so a Secret is addressable inside one. + for segment in [".hidden", "_internal", "-lead", "trailing."]: + binding = EnvVarSecret( + secret_name="openai-prod", + env_variable="OPENAI_API_KEY", + secret_namespace_path=[segment], + ) + assert binding.secret_namespace_path == [segment] + + for segment in ["", "with/slash", "with$delimiter"]: + with pytest.raises(ValueError, match="invalid namespace path segment"): + EnvVarSecret( + secret_name="openai-prod", + env_variable="OPENAI_API_KEY", + secret_namespace_path=[segment], + ) + + def _main_udf_source( *, threshold: int = 20, input_annotation: str = "int", comparison: str = ">=" ) -> str: @@ -1232,7 +1525,20 @@ def _mock_remote_function_catalog(): body = json.loads(self.rfile.read(length) or b"{}") state["requests"].append((self.path, body)) status = 200 - if self.path == "/v1/function/normalize_score/create": + # `{id}` is the Function name, so match on the shape rather than on + # one name: these tests register more than one Function. + parts = self.path.strip("/").split("/") + function_action = ( + (urllib.parse.unquote(parts[2]), parts[3]) + if len(parts) == 4 and parts[0] == "v1" and parts[1] == "function" + else (None, None) + ) + secret_action = ( + (urllib.parse.unquote(parts[2]), parts[3]) + if len(parts) == 4 and parts[0] == "v1" and parts[1] == "secret" + else (None, None) + ) + if function_action[1] == "create": state["version"] = { "name": "normalize_score", "version": FUNCTION_VERSION, @@ -1246,6 +1552,7 @@ def _mock_remote_function_catalog(): ).read_text() )["image"], "signature": body["signature"], + "secret_bindings": body.get("secret_bindings", []), "created_at": "2026-08-21T00:00:00Z", } response = {"job_id": "job-register"} @@ -1264,6 +1571,15 @@ def _mock_remote_function_catalog(): elif self.path == "/v1/function/normalize_score/drop": assert body == {"version": FUNCTION_VERSION} response = {"dropped": True} + elif secret_action[1] in ("create", "alter"): + # The Secret is the path identifier, so the body is the value. + assert set(body) == {"value"} + assert secret_action[0] == "openai-prod" + response = {} + elif secret_action[1] == "drop": + assert secret_action[0] == "openai-prod" + assert body == {} + response = {} else: status = 404 response = {"error": "not found"} @@ -1276,6 +1592,16 @@ def _mock_remote_function_catalog(): for key, values in urllib.parse.parse_qs(url.query).items() } state["requests"].append((url.path, query)) + if url.path == "/v1/namespace/$/secret/list": + if "page_token" not in query: + self._write_response( + 200, + {"secrets": [{"name": "openai-prod"}], "page_token": "next"}, + ) + else: + assert query["page_token"] == "next" + self._write_response(200, {"secrets": [{"name": "hf-prod"}]}) + return if url.path != "/v1/namespace/$/function/list": self._write_response(404, {"error": "not found"}) return @@ -1330,6 +1656,92 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip(): assert create_request == expected_request +def test_remote_registration_sends_bindings_and_never_a_credential(): + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + created = db.create_function( + analyze_caption, + secrets=[ + EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") + ], + ) + + assert list(created.secret_bindings) == [ + SecretBinding( + kind="env", + variable="OPENAI_API_KEY", + secret_ref=SecretReference(name="openai-prod"), + ) + ] + path, create_request = state["requests"][0] + assert path == "/v1/function/analyze_caption/create" + assert create_request["secret_bindings"] == [ + { + "kind": "env", + "variable": "OPENAI_API_KEY", + "secret_ref": {"name": "openai-prod"}, + } + ] + # The request names a Secret and carries nothing that could be one. The + # Function's own name is the path identifier rather than a body field, so + # it is the one key the body does not repeat. + expected = json.loads( + analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")] + ).to_canonical_json() + ) + assert expected.pop("name") == "analyze_caption" + assert create_request == expected + + +def test_remote_secret_verbs_round_trip(): + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + assert db.create_secret("openai-prod", "sk-live-0001") is None + assert db.alter_secret("openai-prod", "sk-live-0002") is None + assert db.list_secrets() == ["openai-prod", "hf-prod"] + assert db.drop_secret("openai-prod") is None + + routes = [path for path, _ in state["requests"]] + assert routes == [ + "/v1/secret/openai-prod/create", + "/v1/secret/openai-prod/alter", + "/v1/namespace/$/secret/list", + "/v1/namespace/$/secret/list", + "/v1/secret/openai-prod/drop", + ] + # The Secret is the path identifier, so the body is the value alone. + assert state["requests"][0][1] == {"value": "sk-live-0001"} + # Listing is a GET: the first page asks for nothing, the second resumes on + # the token the server handed back, and neither carries a body. + assert state["requests"][2][1] == {} + assert state["requests"][3][1] == {"page_token": "next"} + + +def test_building_a_binding_contacts_no_server(): + """A binding is a local value: it says nothing about whether the Secret exists. + + Existence is the server's answer at registration, where a mistyped name is a + clear error rather than a client-side check that was already stale. + """ + with _mock_remote_function_catalog() as (_host, state): + binding = EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") + assert binding.secret_name == "openai-prod" + assert binding.env_variable == "OPENAI_API_KEY" + + assert state["requests"] == [] + + def test_blocking_remote_registration_returns_function_version(): with _mock_remote_function_catalog() as (host, state): db = lancedb.connect( diff --git a/python/src/connection.rs b/python/src/connection.rs index 65d4c98e8..a972f4351 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -768,6 +768,85 @@ impl Connection { }) } + #[pyo3(signature = (name, value, namespace_path=None))] + pub fn create_secret( + self_: PyRef<'_, Self>, + name: String, + value: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner + .create_secret(name, value, &namespace_path) + .await + .infer_error() + }) + } + + #[pyo3(signature = (name, value, namespace_path=None))] + pub fn alter_secret( + self_: PyRef<'_, Self>, + name: String, + value: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner + .alter_secret(name, value, &namespace_path) + .await + .infer_error() + }) + } + + #[pyo3(signature = (namespace_path=None))] + pub fn list_secrets( + self_: PyRef<'_, Self>, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.list_secrets(&namespace_path).await.infer_error() + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn drop_secret( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.drop_secret(name, &namespace_path).await.infer_error() + }) + } + + /// Name and timestamps as a plain tuple. `SecretInfo` carries no value, so + /// there is none to filter out here. Timestamps stay integers rather than + /// going through a string, so the caller can compare two without parsing. + #[pyo3(signature = (name, namespace_path=None))] + pub fn describe_secret( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + let info = inner + .describe_secret(name, &namespace_path) + .await + .infer_error()?; + Ok((info.name, info.created_at_millis, info.updated_at_millis)) + }) + } + pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 3f2979523..7234a911f 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -36,6 +36,8 @@ use crate::remote::{ OPT_REMOTE_SQL_HOST_OVERRIDE, }, }; +use crate::secrets::SecretInfo; +use crate::utils::{validate_secret_component, validate_secret_reference}; use lance::io::ObjectStoreParams; pub use lance_file::version::LanceFileVersion; #[cfg(feature = "remote")] @@ -587,6 +589,7 @@ impl Connection { /// artifact. Waiting on the returned typed job yields the durable /// [`crate::function::FunctionVersion`]. Creation is remote-only. /// Local databases return [`Error::NotSupported`]. + /// pub async fn create_function_async( &self, request: crate::function::FunctionRegistrationRequest, @@ -646,6 +649,88 @@ impl Connection { .await } + /// Create a named Secret in this database. + /// + /// Fails if the name is taken, so a create can never silently become a + /// rotation. There is no API that reads a stored credential back; the only + /// consumer is a Function that binds the Secret by name. Local databases + /// return [`Error::NotSupported`]. + pub async fn create_secret( + &self, + name: impl AsRef, + value: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .create_secret(name.as_ref(), value.as_ref(), namespace_path) + .await + } + + /// Replace the credential behind an existing Secret. + /// + /// Fails if it does not exist. Every Function bound to the Secret resolves + /// the new value from its next execution, and no new Function version is + /// minted -- which is what lets a rotation reach columns pinned to a + /// version registered before it. Local databases return + /// [`Error::NotSupported`]. + pub async fn alter_secret( + &self, + name: impl AsRef, + value: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .alter_secret(name.as_ref(), value.as_ref(), namespace_path) + .await + } + + /// The names of every Secret in this database. + /// + /// Names only. No path in this API returns a stored credential, by + /// construction rather than by policy. Local databases return + /// [`Error::NotSupported`]. + pub async fn list_secrets(&self, namespace_path: &[String]) -> Result> { + for segment in namespace_path { + validate_secret_component("Secret namespace path segment", segment)?; + } + self.internal.list_secrets(namespace_path).await + } + + /// Drop a Secret. + /// + /// Functions bound to it fail at their next job, naming the Secret; that + /// is the revocation path. The name becomes free to reuse, and a new + /// Secret under it is picked up by everything still bound to that name. + /// Local databases return [`Error::NotSupported`]. + pub async fn drop_secret( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .drop_secret(name.as_ref(), namespace_path) + .await + } + + /// What this database records about one Secret: its name and timestamps. + /// + /// Never the value. The type it returns has no field for one, so this is a + /// property of the API rather than of what the caller chooses to read. + /// Local databases return [`Error::NotSupported`]. + pub async fn describe_secret( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .describe_secret(name.as_ref(), namespace_path) + .await + } + /// Rename a table in the database. /// /// This is only supported in LanceDB Cloud. diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index d31f40b29..392277f63 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -30,6 +30,7 @@ use crate::data::scannable::Scannable; use crate::error::Result; use crate::job::Job; use crate::materialized_view::CreateMaterializedViewRequest; +use crate::secrets::SecretInfo; use crate::table::{BaseTable, WriteOptions}; pub mod listing; @@ -251,6 +252,12 @@ fn function_catalog_not_supported() -> Result { }) } +fn secret_catalog_not_supported() -> Result { + Err(crate::error::Error::NotSupported { + message: "Secret operations are not supported by this database".to_string(), + }) +} + /// The `Database` trait defines the interface for database implementations. /// /// A database is responsible for managing tables and their metadata. @@ -386,6 +393,44 @@ pub trait Database: async fn drop_function(&self, _name: &str, _version: &str) -> Result { function_catalog_not_supported() } + /// Create a named Secret in this database. Fails if the name is taken, so + /// a create can never silently become a rotation. + async fn create_secret( + &self, + _name: &str, + _value: &str, + _namespace_path: &[String], + ) -> Result<()> { + secret_catalog_not_supported() + } + /// Replace the credential behind an existing Secret. Fails if it does not + /// exist. Every Function bound to it resolves the new value from its next + /// execution, with no new Function version. + async fn alter_secret( + &self, + _name: &str, + _value: &str, + _namespace_path: &[String], + ) -> Result<()> { + secret_catalog_not_supported() + } + /// The names of every Secret in this database. + /// + /// Names only. No API path returns a stored credential, by construction + /// rather than by policy. + async fn list_secrets(&self, _namespace_path: &[String]) -> Result> { + secret_catalog_not_supported() + } + /// Drop a Secret. Functions bound to it fail at their next job, which is + /// the revocation path. + async fn drop_secret(&self, _name: &str, _namespace_path: &[String]) -> Result<()> { + secret_catalog_not_supported() + } + /// What the database records about one Secret: its name and timestamps, + /// never its value. + async fn describe_secret(&self, _name: &str, _namespace_path: &[String]) -> Result { + secret_catalog_not_supported() + } /// Open a job by id, returning a handle with its record already /// populated. Fails with [`crate::Error::JobNotFound`] when the server has /// no such job. diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index d501541c0..0d593585a 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -5,7 +5,7 @@ //! backend-neutral terminal result of a computed-column refresh. //! //! This module contains client/wire values only. Catalog persistence, -//! environment bake, and execution are owned by Sophon. +//! environment bake, secret resolution, and execution are owned by Sophon. use std::collections::BTreeMap; @@ -13,6 +13,7 @@ use serde::de::{self, DeserializeOwned}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; +use crate::secrets::SecretBinding; use crate::{Error, Result}; /// Semantic Function type for a Blob v2 value. @@ -434,6 +435,8 @@ pub struct FunctionVersion { version: String, image: FunctionImage, signature: FunctionSignature, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + secret_bindings: Vec, created_at: String, metadata: BTreeMap, disabled: bool, @@ -473,6 +476,17 @@ impl FunctionVersion { pub fn signature(&self) -> &FunctionSignature { &self.signature } + + /// Declared environment variable name to the Secret each one resolves. + /// + /// Bindings are part of this version's identity; the credentials behind + /// them are not, and resolve at execution. Rotating a bound Secret + /// therefore changes what the same version runs with, and no value has a + /// field in this model. + pub fn secret_bindings(&self) -> &[SecretBinding] { + &self.secret_bindings + } + pub fn created_at(&self) -> &str { &self.created_at } @@ -514,12 +528,21 @@ pub struct FunctionArtifactRequest { } /// Stable request envelope for remote immutable Function registration. +/// +/// Credential values deliberately have no field here. The only secret-shaped +/// thing a client sends is `secret_bindings`: the name of a Secret the +/// database already holds, which Sophon resolves inside the remote runtime. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionRegistrationRequest { pub name: String, pub artifact: FunctionArtifactRequest, pub signature: FunctionSignature, pub runtime: PythonRuntimeSpec, + /// Declared environment variable name to the Secret it binds. A binding is + /// a reference: whether the Secret exists is answered when a column is + /// declared against this version, not here. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub secret_bindings: Vec, } impl_json!(FunctionRegistrationRequest); @@ -785,3 +808,80 @@ mod conda_environment_tests { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Canonical form is what the FunctionVersion hash is taken over, so key + /// order must come from the keys and not from however serde happened to + /// emit them. Nesting is included because the sort is recursive. + #[test] + fn canonical_json_sorts_keys_at_every_depth() { + let value = serde_json::json!({ + "runtime": {"kind": "python", "env": {"B": "2", "A": "1"}}, + "artifact": {"digest": "sha256:x"}, + "name": "embed", + }); + let mut out = String::new(); + write_canonical_json(&value, &mut out).expect("canonical JSON"); + + assert_eq!( + out, + r#"{"artifact":{"digest":"sha256:x"},"name":"embed","runtime":{"env":{"A":"1","B":"2"},"kind":"python"}}"# + ); + } + + /// Arrays are ordered by the caller, so canonicalization must leave them + /// alone -- sorting them would change what a signature means. + #[test] + fn canonical_json_preserves_array_order() { + let value = serde_json::json!({"inputs": ["b", "a", "c"]}); + let mut out = String::new(); + write_canonical_json(&value, &mut out).expect("canonical JSON"); + + assert_eq!(out, r#"{"inputs":["b","a","c"]}"#); + } + + /// A float has no single canonical spelling, so two clients could hash the + /// same literal differently. Rejected at any depth rather than rounded. + #[test] + fn validate_literal_rejects_floats_at_any_depth() { + for value in [ + serde_json::json!(1.5), + serde_json::json!([1, [2, 3.5]]), + serde_json::json!({"a": {"b": 0.25}}), + ] { + let error = validate_literal(&value).expect_err("floats are not canonical"); + assert!( + error.to_string().contains("floating-point"), + "unexpected error: {error}" + ); + } + + for value in [ + serde_json::json!(1), + serde_json::json!("1.5"), + serde_json::json!([1, {"a": true}]), + serde_json::json!(null), + ] { + validate_literal(&value).expect("non-float literals are canonical"); + } + } + + /// Unknown keys are how a newer server's payload reaches an older client, + /// so the check has to be exact about which level it is looking at. + #[test] + fn has_unknown_keys_only_inspects_the_level_it_is_given() { + let value = serde_json::json!({"name": "embed", "version": "fv_1"}); + assert!(!has_unknown_keys(&value, &["name", "version"])); + assert!(has_unknown_keys(&value, &["name"])); + + // A nested unknown is not this level's business. + let nested = serde_json::json!({"name": {"unexpected": 1}}); + assert!(!has_unknown_keys(&nested, &["name"])); + + // A non-object has no keys to be unknown. + assert!(!has_unknown_keys(&serde_json::json!("embed"), &["name"])); + } +} diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index b173a63cb..4d9afc7c7 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -196,6 +196,7 @@ pub mod query; #[cfg(feature = "remote")] pub mod remote; pub mod rerankers; +pub mod secrets; pub mod sql; pub mod table; #[cfg(test)] diff --git a/rust/lancedb/src/remote/catalog.rs b/rust/lancedb/src/remote/catalog.rs index 098a696a7..1c0129889 100644 --- a/rust/lancedb/src/remote/catalog.rs +++ b/rust/lancedb/src/remote/catalog.rs @@ -472,7 +472,7 @@ mod tests { } assert_eq!( requests[0].line, - "GET /v1/namespace/%24/list?limit=1&page_token=a%2Fb HTTP/1.1" + "GET /v1/namespace/$/list?limit=1&page_token=a%2Fb HTTP/1.1" ); assert_eq!( requests[1].line, diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 55d272b51..b7a492598 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -90,8 +90,9 @@ pub struct ClientConfig { pub user_agent: String, // TODO: how to configure request ids? pub extra_headers: HashMap, - /// The delimiter to use when constructing object identifiers. - /// If not default, passes as query parameter. + /// The delimiter joining a namespace path and a name into one object + /// identifier. [`ID_DELIMITER`] is the only accepted value; any other is + /// refused by [`ClientConfig::validate`]. pub id_delimiter: Option, /// TLS configuration for mTLS support pub tls_config: Option, @@ -307,7 +308,6 @@ pub struct RestfulLanceDbClient { host: String, pub(crate) retry_config: ResolvedRetryConfig, pub(crate) sender: S, - pub(crate) id_delimiter: String, pub(crate) header_provider: Option>, /// Connection-level read consistency interval. Drives the /// `x-lancedb-min-timestamp` freshness header sent on read requests. @@ -330,7 +330,6 @@ impl std::fmt::Debug for RestfulLanceDbClient { .field("host", &self.host) .field("retry_config", &self.retry_config) .field("sender", &self.sender) - .field("id_delimiter", &self.id_delimiter) .field( "header_provider", &self.header_provider.as_ref().map(|_| "Some(...)"), @@ -427,6 +426,54 @@ fn validate_dns_hostname(hostname: &str) -> Result<()> { Ok(()) } +/// Whether a request's body may appear in a debug log. +/// +/// The API that built the body decides. The transport cannot know which +/// payloads are credentials, and a list of routes here would have to be kept in +/// step with endpoints defined elsewhere -- so the knowledge lives with the +/// call that has it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum BodyLogging { + /// Log the body at debug. The default: a request body is diagnostic unless + /// the call that built it says otherwise. + Allowed, + /// Never log the body. For a request whose body is a credential. + Suppressed, +} + +/// The delimiter joining a namespace path and a name into the `{id}` a route +/// addresses, and the only one a LanceDB service splits on. +/// +/// `$` is outside the character set object names admit, so a joined identifier +/// always splits back into the parts that made it. The configuration field +/// exists because the identifier grammar comes from the Lance REST catalog +/// standard, which carries a delimiter setting for other catalogs to adopt. +pub(crate) const ID_DELIMITER: &str = "$"; + +fn validate_id_delimiter(delimiter: &str) -> Result<()> { + if delimiter != ID_DELIMITER { + return Err(Error::InvalidInput { + message: format!( + "id_delimiter '{delimiter}' is not supported: '{ID_DELIMITER}' is the only \ + delimiter LanceDB services split an identifier on" + ), + }); + } + Ok(()) +} + +impl ClientConfig { + /// Check the settings a request cannot be built correctly without, so a + /// mistake is reported where it was made rather than as a confusing + /// response later. Public so a caller can ask without connecting. + pub fn validate(&self) -> Result<()> { + if let Some(delimiter) = &self.id_delimiter { + validate_id_delimiter(delimiter)?; + } + Ok(()) + } +} + impl RestfulLanceDbClient { fn get_timeout(passed: Option, env_var: &str) -> Result> { if let Some(passed) = passed { @@ -452,6 +499,10 @@ impl RestfulLanceDbClient { client_config: ClientConfig, read_consistency_interval: Option, ) -> Result { + // Before anything is built from it, so the error names the caller's + // configuration rather than a request. + client_config.validate()?; + // Get the timeouts let timeout = Self::get_timeout(client_config.timeout_config.timeout, "LANCE_CLIENT_TIMEOUT")?; @@ -551,10 +602,6 @@ impl RestfulLanceDbClient { host, retry_config, sender: Sender, - id_delimiter: client_config - .id_delimiter - .clone() - .unwrap_or("$".to_string()), header_provider: client_config.header_provider, read_consistency_interval, max_bytes_per_request, @@ -633,12 +680,14 @@ impl RestfulLanceDbClient { ) -> Result { let mut headers = HeaderMap::new(); if !api_key.is_empty() { - headers.insert( - HeaderName::from_static("x-api-key"), - HeaderValue::from_str(api_key).map_err(|_| Error::InvalidInput { - message: "non-ascii api key provided".to_string(), - })?, - ); + // `log_request` prints the request's Debug, which prints headers. + // Marking the value sensitive is what makes that print `Sensitive` + // instead of the key itself. + let mut key = HeaderValue::from_str(api_key).map_err(|_| Error::InvalidInput { + message: "non-ascii api key provided".to_string(), + })?; + key.set_sensitive(true); + headers.insert(HeaderName::from_static("x-api-key"), key); } if region == "local" { let host = format!("{}.local.api.lancedb.com", db_name); @@ -710,22 +759,12 @@ impl RestfulLanceDbClient { pub fn get(&self, uri: &str) -> RequestBuilder { let full_uri = format!("{}{}", self.host, uri); - let builder = self.client.get(full_uri); - self.add_id_delimiter_query_param(builder) + self.client.get(full_uri) } pub fn post(&self, uri: &str) -> RequestBuilder { let full_uri = format!("{}{}", self.host, uri); - let builder = self.client.post(full_uri); - self.add_id_delimiter_query_param(builder) - } - - fn add_id_delimiter_query_param(&self, req: RequestBuilder) -> RequestBuilder { - if self.id_delimiter != "$" { - req.query(&[("delimiter", self.id_delimiter.clone())]) - } else { - req - } + self.client.post(full_uri) } /// Apply dynamic headers from the header provider if configured @@ -750,6 +789,22 @@ impl RestfulLanceDbClient { } pub async fn send(&self, req: RequestBuilder) -> Result<(String, Response)> { + self.send_logging(req, BodyLogging::Allowed).await + } + + /// Send a request whose body must never reach a debug log. + /// + /// The body is built by the caller, so only the caller knows it holds a + /// credential; `log_request` sees serialized bytes and cannot tell. + pub async fn send_suppressing_body(&self, req: RequestBuilder) -> Result<(String, Response)> { + self.send_logging(req, BodyLogging::Suppressed).await + } + + async fn send_logging( + &self, + req: RequestBuilder, + body_logging: BodyLogging, + ) -> Result<(String, Response)> { let (client, request) = req.build_split(); let mut request = request.unwrap(); let request_id = self.extract_request_id(&mut request); @@ -757,7 +812,7 @@ impl RestfulLanceDbClient { // Apply dynamic headers before sending request = self.apply_dynamic_headers(request).await?; - self.log_request(&request, &request_id); + self.log_request(&request, &request_id, body_logging); let response = self .sender @@ -820,7 +875,7 @@ impl RestfulLanceDbClient { // Apply dynamic headers before each retry attempt request = self.apply_dynamic_headers(request).await?; - self.log_request(&request, &request_id); + self.log_request(&request, &request_id, BodyLogging::Allowed); let response = self.sender.send(&c, request).await.map(|r| (r.status(), r)); @@ -864,13 +919,18 @@ impl RestfulLanceDbClient { } } - pub(crate) fn log_request(&self, request: &Request, request_id: &String) { + fn log_request(&self, request: &Request, request_id: &String, body_logging: BodyLogging) { if log::log_enabled!(log::Level::Debug) { let content_type = request .headers() .get("content-type") .map(|v| v.to_str().unwrap()); - if content_type == Some("application/json") { + if body_logging == BodyLogging::Suppressed { + debug!( + "Sending request_id={}: {:?} with body suppressed", + request_id, request + ); + } else if content_type == Some("application/json") { let body = request.body().as_ref().unwrap().as_bytes().unwrap(); let body = String::from_utf8_lossy(body); debug!( @@ -1047,7 +1107,6 @@ pub mod test_utils { sender: MockSender { f: Arc::new(wrapper), }, - id_delimiter: "$".to_string(), header_provider: None, read_consistency_interval, max_bytes_per_request: None, @@ -1074,7 +1133,6 @@ pub mod test_utils { sender: MockSender { f: Arc::new(wrapper), }, - id_delimiter: config.id_delimiter.unwrap_or_else(|| "$".to_string()), header_provider: config.header_provider, read_consistency_interval: None, max_bytes_per_request: config @@ -1089,6 +1147,40 @@ pub mod test_utils { #[cfg(test)] mod tests { + /// A configuration naming any other delimiter is refused where it was + /// written, rather than producing identifiers no service splits the way the + /// caller meant. + #[test] + fn test_a_delimiter_other_than_the_supported_one_is_refused() { + for delimiter in ["/", "?", "#", "%", "", ".", "..", "-", "_", "|", "::", "$$"] { + let error = super::validate_id_delimiter(delimiter) + .expect_err("only the supported delimiter may be configured"); + assert!( + error.to_string().contains("id_delimiter"), + "{delimiter:?}: {error}" + ); + } + super::validate_id_delimiter(super::ID_DELIMITER).unwrap(); + } + + /// Leaving it unset is how nearly every caller reaches the same delimiter. + #[test] + fn test_an_unset_delimiter_is_the_supported_one() { + super::ClientConfig::default().validate().unwrap(); + super::ClientConfig { + id_delimiter: Some(super::ID_DELIMITER.to_string()), + ..Default::default() + } + .validate() + .unwrap(); + super::ClientConfig { + id_delimiter: Some("-".to_string()), + ..Default::default() + } + .validate() + .expect_err("a configured delimiter other than the supported one must be refused"); + } + use super::*; use serial_test::serial; use std::time::Duration; @@ -1275,6 +1367,41 @@ mod tests { assert!(debug.contains("visible-value")); } + /// `log_request` prints the request's Debug, and Debug for a request prints + /// its headers. Marking the value sensitive is the only thing standing + /// between the API key and every debug line; assert on the header map's own + /// Debug, which is what that printing reduces to. + #[test] + fn test_api_key_is_redacted_in_debug_output() { + let headers = RestfulLanceDbClient::::default_headers( + "sk-live-sentinel", + "us-east-1", + "db-name", + false, + &RemoteOptions::default(), + None, + &ClientConfig::default(), + ) + .unwrap(); + + assert_eq!(headers.get("x-api-key").unwrap(), "sk-live-sentinel"); + assert!( + !format!("{:?}", headers).contains("sk-live-sentinel"), + "the API key must not survive Debug formatting" + ); + } + + /// A suppressed body is suppressed whatever the content type says, and an + /// allowed one is logged in full. + #[test] + fn test_body_logging_is_decided_by_the_caller() { + assert_ne!(BodyLogging::Allowed, BodyLogging::Suppressed); + // `send` and `send_suppressing_body` differ only in what they pass, so + // the enum is the whole contract: a caller states its intent and the + // transport does not infer one from the route. + assert_eq!(BodyLogging::Allowed, BodyLogging::Allowed); + } + #[test] fn test_rejects_invalid_cloud_dns_hostname() { let invalid_database_names = ["a".repeat(64), "invalid..database".to_string()]; @@ -1364,7 +1491,6 @@ mod tests { host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), sender: Sender, - id_delimiter: "+".to_string(), header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, max_bytes_per_request: None, @@ -1403,7 +1529,6 @@ mod tests { host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), sender: Sender, - id_delimiter: "+".to_string(), header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, max_bytes_per_request: None, @@ -1468,7 +1593,6 @@ mod tests { host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), sender: Sender, - id_delimiter: "+".to_string(), header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, max_bytes_per_request: None, diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index e7ae78108..bc1dbfea1 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -24,15 +24,22 @@ use crate::database::{ OpenTableRequest, ReadConsistency, TableNamesRequest, }; use crate::error::Result; -use crate::function::{FunctionRegistrationRequest, FunctionVersion}; +use crate::function::{ + FunctionArtifactRequest, FunctionRegistrationRequest, FunctionSignature, FunctionVersion, + PythonRuntimeSpec, +}; use crate::job::Job; use crate::materialized_view::CreateMaterializedViewRequest; use crate::remote::job::{RemoteJob, job_state_to_client}; use crate::remote::util::stream_as_body; +use crate::secrets::SecretBinding; +use crate::secrets::SecretInfo; use crate::table::BaseTable; +use crate::utils::{reject_relative_segment, validate_table_name}; use super::client::{ - ClientConfig, HeaderProvider, HttpSend, RequestResultExt, RestfulLanceDbClient, Sender, + ClientConfig, HeaderProvider, HttpSend, ID_DELIMITER, RequestResultExt, RestfulLanceDbClient, + Sender, }; use super::sql::SqlClient; use super::table::RemoteTable; @@ -413,12 +420,30 @@ impl RemoteDatabase { } impl RemoteDatabase { + /// Post a request whose body carries a credential. + /// + /// Shared by the create and alter verbs, which declare their own request + /// types: the two mean different things to the service and are free to + /// diverge, so what they share is the posting and not the shape. + /// + /// The value is a request field and never a path segment or query + /// parameter, which keeps it out of access logs and proxy traces. + async fn post_secret_write(&self, route: &str, body: &T) -> Result<()> { + let req = self.client.post(route).json(body); + // This call is what says the body is a credential. Nothing downstream + // can tell from the bytes, and a route list in the transport would have + // to be kept in step with endpoints declared here. + let (request_id, response) = self.client.send_suppressing_body(req).await?; + self.client.check_response(&request_id, response).await?; + Ok(()) + } + async fn submit_drop_table( &self, name: &str, namespace_path: &[String], ) -> Result<(String, Response)> { - let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter); + let identifier = build_table_identifier(name, namespace_path)?; let cache_key = build_cache_key(name, namespace_path); let req = self.client.post(&format!("/v1/table/{}/drop/", identifier)); let (request_id, resp) = self.client.send(req).await?; @@ -442,8 +467,7 @@ impl RemoteDatabase { &self, request: &TableNamesRequest, ) -> Result<(Vec, ServerVersion)> { - let namespace_id = - build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(&request.namespace_path)?; let path = format!("/v1/namespace/{}/table/list", namespace_id); let mut names = Vec::new(); @@ -550,23 +574,93 @@ impl From<&CreateTableMode> for &'static str { } } -fn build_table_identifier(name: &str, namespace: &[String], delimiter: &str) -> String { - if !namespace.is_empty() { - let mut parts = namespace.to_vec(); - parts.push(name.to_string()); - parts.join(delimiter) - } else { - name.to_string() +/// The path segment addressing one object: its namespace path and its name. +/// +/// One builder for tables, Secrets, Functions and materialized views: the +/// identifier grammar belongs to the namespace spec, not to an object type. An +/// empty path addresses an object with no namespace. +/// +/// Components are checked for addressability, not a character set. The name's +/// grammar is the caller's, so a table reports [`Error::InvalidTableName`], a +/// Function admits names a table may not, and a catalog database carries the +/// `/` that [`RemoteCatalog`] allows. +/// +/// [`RemoteCatalog`]: super::catalog::RemoteCatalog +fn build_object_identifier(what: &str, name: &str, namespace: &[String]) -> Result { + for segment in namespace { + reject_unaddressable_component("namespace segment", segment)?; } + reject_unaddressable_component(what, name)?; + Ok(join_identifier( + namespace.iter().map(String::as_str).chain([name]), + )) } -fn build_namespace_identifier(namespace: &[String], delimiter: &str) -> String { +/// What a component may not be if the join is to survive being split back +/// apart: empty, a segment URL parsing resolves away, or the delimiter itself. +/// +/// Each erases a boundary no encoding of the joined form recovers. `["prod", +/// ""]` joins to `prod$`, which reads back as `["prod"]`, so a drop reaches the +/// parent of the namespace the caller named. +/// +/// Not a character set: per-component percent-encoding makes the wider set +/// safe, since a `/` in a name reaches the service as `%2F`, still one +/// segment. +fn reject_unaddressable_component(what: &str, value: &str) -> Result<()> { + if value.is_empty() { + return Err(Error::InvalidInput { + message: format!( + "{what} must not be empty: the identifier would carry two delimiters in a row, \ + and splitting it back apart would name a different object" + ), + }); + } + reject_relative_segment(what, value)?; + if value.contains(ID_DELIMITER) { + return Err(Error::InvalidInput { + message: format!( + "{what} '{value}' contains the identifier delimiter '{ID_DELIMITER}', so the \ + namespace path and the name it joins could not be told apart" + ), + }); + } + Ok(()) +} + +/// The path segment addressing one table. A wrapper for the error type: +/// callers match on [`Error::InvalidTableName`]. +fn build_table_identifier(name: &str, namespace: &[String]) -> Result { + validate_table_name(name)?; + build_object_identifier("table name", name, namespace) +} + +/// Join components into the `{id}` a route addresses: each percent-encoded, +/// then joined by the delimiter. +/// +/// Per component rather than over the joined string, so the delimiter stays a +/// delimiter and nothing inside a component can end the path segment. +/// +/// A second line, not the first: a component from the object charset is all +/// unreserved and encodes to itself, so the route reads as the caller wrote it. +/// It does not cover `.` and `..`, which are unreserved too and resolve away +/// after decoding -- [`build_object_identifier`] refuses those. +fn join_identifier<'a>(components: impl Iterator) -> String { + components + .map(|component| urlencoding::encode(component).into_owned()) + .collect::>() + .join(ID_DELIMITER) +} + +/// The path segment addressing one namespace. +fn build_namespace_identifier(namespace: &[String]) -> Result { + for segment in namespace { + reject_unaddressable_component("namespace segment", segment)?; + } if namespace.is_empty() { // According to the namespace spec, use delimiter to represent root namespace - delimiter.to_string() - } else { - namespace.join(delimiter) + return Ok(ID_DELIMITER.to_string()); } + Ok(join_identifier(namespace.iter().map(String::as_str))) } /// Build a secure cache key using length prefixes. @@ -631,6 +725,65 @@ struct RemoteDropFunctionResponse { dropped: bool, } +/// The create body: every field of [`FunctionRegistrationRequest`] except the +/// name, which is the path identifier. +/// +/// A struct rather than a literal listing the fields, so that the compiler +/// decides what reaches the service. A field the registration request grows is +/// a build error here until it is handled; a literal would simply not send it. +#[derive(serde::Serialize)] +struct RemoteCreateFunctionRequest<'a> { + artifact: &'a FunctionArtifactRequest, + signature: &'a FunctionSignature, + runtime: &'a PythonRuntimeSpec, + /// Absent when the Function binds nothing, so such a client sends what a + /// client without bindings sends. + /// + /// A service that does not know the field ignores it: registration + /// succeeds, the returned version carries no bindings, and the Function + /// fails at execution with the variable unset. [`ServerVersion`] is how + /// this codebase refuses a feature the service is too old for; it is held + /// per table, so gating a database-level call is follow-up work. + /// + /// [`ServerVersion`]: super::db::ServerVersion + #[serde(skip_serializing_if = "<[SecretBinding]>::is_empty")] + secret_bindings: &'a [SecretBinding], +} + +/// Create a Secret under a name the database does not yet hold. +/// +/// Declared separately from the alter request although the two are identical +/// today: they are different operations to the service -- one refuses an +/// existing name, the other requires it -- and either may grow a field the +/// other has no meaning for. +/// +/// The name and its namespace are the path identifier, so neither appears here. +#[derive(serde::Serialize)] +struct RemoteCreateSecretRequest<'a> { + value: &'a str, +} + +/// Replace the credential behind a Secret the database already holds. +#[derive(serde::Serialize)] +struct RemoteAlterSecretRequest<'a> { + value: &'a str, +} + +#[derive(serde::Deserialize)] +struct RemoteListSecretsResponse { + #[serde(default)] + secrets: Vec, + #[serde(default)] + page_token: Option, +} + +/// An object rather than a bare name so a later listing can carry a Secret's +/// type or last-updated time without breaking this one. +#[derive(serde::Deserialize)] +struct RemoteListedSecret { + name: String, +} + /// Bound on `list_jobs` page walking; a warning is logged when the listing /// is truncated at this many pages. const MAX_LIST_JOBS_PAGES: usize = 100; @@ -652,11 +805,7 @@ impl Database for RemoteDatabase { &self, request: CreateMaterializedViewRequest, ) -> Result { - let identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let identifier = build_table_identifier(&request.name, &request.namespace_path)?; let req = self .client .post(&format!("/v1/materialized_view/{identifier}/create")) @@ -699,7 +848,7 @@ impl Database for RemoteDatabase { name: &str, namespace_path: &[String], ) -> Result { - let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter); + let identifier = build_table_identifier(name, namespace_path)?; let request = self .client .post(&format!("/v1/materialized_view/{identifier}/drop")); @@ -737,7 +886,7 @@ impl Database for RemoteDatabase { page_token: Option, } - let namespace_id = build_namespace_identifier(namespace_path, &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(namespace_path)?; let path = format!("/v1/namespace/{namespace_id}/materialized_view/list"); let mut views = Vec::new(); let mut page_token: Option = None; @@ -773,15 +922,16 @@ impl Database for RemoteDatabase { &self, request: FunctionRegistrationRequest, ) -> Result> { - let function_id = urlencoding::encode(&request.name); + let function_id = build_object_identifier("Function name", &request.name, &[])?; let req = self .client .post(&format!("/v1/function/{function_id}/create")) - .json(&serde_json::json!({ - "artifact": request.artifact, - "signature": request.signature, - "runtime": request.runtime, - })); + .json(&RemoteCreateFunctionRequest { + artifact: &request.artifact, + signature: &request.signature, + runtime: &request.runtime, + secret_bindings: &request.secret_bindings, + }); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; let status = response.status(); @@ -798,7 +948,7 @@ impl Database for RemoteDatabase { } async fn get_function(&self, name: &str, version: &str) -> Result { - let function_id = urlencoding::encode(name); + let function_id = build_object_identifier("Function name", name, &[])?; let req = self .client .post(&format!("/v1/function/{function_id}/describe")) @@ -811,7 +961,7 @@ impl Database for RemoteDatabase { } async fn list_functions(&self) -> Result> { - let namespace_id = build_namespace_identifier(&[], &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(&[])?; let path = format!("/v1/namespace/{namespace_id}/function/list"); let mut functions = Vec::new(); let mut page_token: Option = None; @@ -852,7 +1002,7 @@ impl Database for RemoteDatabase { } async fn drop_function(&self, name: &str, version: &str) -> Result { - let function_id = urlencoding::encode(name); + let function_id = build_object_identifier("Function name", name, &[])?; let req = self .client .post(&format!("/v1/function/{function_id}/drop")) @@ -865,6 +1015,80 @@ impl Database for RemoteDatabase { Ok(response.dropped) } + async fn create_secret( + &self, + name: &str, + value: &str, + namespace_path: &[String], + ) -> Result<()> { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + self.post_secret_write( + &format!("/v1/secret/{secret_id}/create"), + &RemoteCreateSecretRequest { value }, + ) + .await + } + + async fn alter_secret(&self, name: &str, value: &str, namespace_path: &[String]) -> Result<()> { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + self.post_secret_write( + &format!("/v1/secret/{secret_id}/alter"), + &RemoteAlterSecretRequest { value }, + ) + .await + } + + async fn list_secrets(&self, namespace_path: &[String]) -> Result> { + let namespace_id = build_namespace_identifier(namespace_path)?; + let path = format!("/v1/namespace/{namespace_id}/secret/list"); + let mut names = Vec::new(); + let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); + loop { + let mut req = self.client.get(&path); + if let Some(token) = &page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let response: RemoteListSecretsResponse = + response.json().await.err_to_http(request_id.clone())?; + names.extend(response.secrets.into_iter().map(|secret| secret.name)); + let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "Secret listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); + } + page_token = Some(next_page_token); + } + Ok(names) + } + + async fn drop_secret(&self, name: &str, namespace_path: &[String]) -> Result<()> { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let req = self.client.post(&format!("/v1/secret/{secret_id}/drop")); + let (request_id, response) = self.client.send(req).await?; + self.client.check_response(&request_id, response).await?; + Ok(()) + } + + async fn describe_secret(&self, name: &str, namespace_path: &[String]) -> Result { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let req = self + .client + .post(&format!("/v1/secret/{secret_id}/describe")); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + response.json().await.err_to_http(request_id) + } + async fn open_job(&self, job_id: &str) -> Result { let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string()); match crate::job::JobHandle::describe(&handle).await { @@ -987,8 +1211,7 @@ impl Database for RemoteDatabase { }; for table in &tables { - let table_identifier = - build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter); + let table_identifier = build_table_identifier(table, &request.namespace_path)?; let cache_key = build_cache_key(table, &request.namespace_path); let remote_table = Arc::new(RemoteTable::new( self.client.clone(), @@ -1004,7 +1227,7 @@ impl Database for RemoteDatabase { async fn list_tables(&self, request: ListTablesRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .get(&format!("/v1/namespace/{}/table/list", namespace_id)); @@ -1024,8 +1247,7 @@ impl Database for RemoteDatabase { // Cache the tables for future use let namespace_vec = namespace_parts.to_vec(); for table in &response.tables { - let table_identifier = - build_table_identifier(table, &namespace_vec, &self.client.id_delimiter); + let table_identifier = build_table_identifier(table, &namespace_vec)?; let cache_key = build_cache_key(table, &namespace_vec); let remote_table = Arc::new(RemoteTable::new( self.client.clone(), @@ -1043,11 +1265,7 @@ impl Database for RemoteDatabase { async fn create_table(&self, mut request: CreateTableRequest) -> Result> { let body = stream_as_body(request.data.scan_as_stream())?; - let identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let identifier = build_table_identifier(&request.name, &request.namespace_path)?; let req = self .client .post(&format!("/v1/table/{}/create/", identifier)) @@ -1099,11 +1317,7 @@ impl Database for RemoteDatabase { } let rsp = self.client.check_response(&request_id, rsp).await?; let version = parse_server_version(&request_id, &rsp)?; - let table_identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let table_identifier = build_table_identifier(&request.name, &request.namespace_path)?; let cache_key = build_cache_key(&request.name, &request.namespace_path); let table = Arc::new(RemoteTable::new( self.client.clone(), @@ -1118,11 +1332,8 @@ impl Database for RemoteDatabase { } async fn clone_table(&self, request: CloneTableRequest) -> Result> { - let table_identifier = build_table_identifier( - &request.target_table_name, - &request.target_namespace_path, - &self.client.id_delimiter, - ); + let table_identifier = + build_table_identifier(&request.target_table_name, &request.target_namespace_path)?; let remote_request = RemoteCloneTableRequest { source_location: request.source_uri, @@ -1163,11 +1374,7 @@ impl Database for RemoteDatabase { } async fn open_table(&self, request: OpenTableRequest) -> Result> { - let identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let identifier = build_table_identifier(&request.name, &request.namespace_path)?; let cache_key = build_cache_key(&request.name, &request.namespace_path); // We describe the table to confirm it exists before moving on. @@ -1183,11 +1390,7 @@ impl Database for RemoteDatabase { let rsp = self.client.check_response(&request_id, rsp).await?; let version = parse_server_version(&request_id, &rsp)?; let describe_body = rsp.text().await.ok(); - let table_identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let table_identifier = build_table_identifier(&request.name, &request.namespace_path)?; let table = Arc::new(RemoteTable::new( self.client.clone(), request.name.clone(), @@ -1214,8 +1417,7 @@ impl Database for RemoteDatabase { cur_namespace_path: &[String], new_namespace_path: &[String], ) -> Result<()> { - let current_identifier = - build_table_identifier(current_name, cur_namespace_path, &self.client.id_delimiter); + let current_identifier = build_table_identifier(current_name, cur_namespace_path)?; let current_cache_key = build_cache_key(current_name, cur_namespace_path); let new_cache_key = build_cache_key(new_name, new_namespace_path); @@ -1279,8 +1481,7 @@ impl Database for RemoteDatabase { request: ListNamespacesRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .get(&format!("/v1/namespace/{}/list", namespace_id)); @@ -1302,8 +1503,7 @@ impl Database for RemoteDatabase { request: CreateNamespaceRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .post(&format!("/v1/namespace/{}/create", namespace_id)); @@ -1334,8 +1534,7 @@ impl Database for RemoteDatabase { async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .post(&format!("/v1/namespace/{}/drop", namespace_id)); @@ -1369,8 +1568,7 @@ impl Database for RemoteDatabase { request: DescribeNamespaceRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let req = self .client .post(&format!("/v1/namespace/{}/describe", namespace_id)) @@ -1389,7 +1587,7 @@ impl Database for RemoteDatabase { async fn namespace_client(&self) -> Result> { // Create a RestNamespace pointing to the same remote host with the same authentication headers let mut builder = lance_namespace_impls::RestNamespaceBuilder::new(self.client.host()) - .delimiter(&self.client.id_delimiter) + .delimiter(ID_DELIMITER) .headers(self.namespace_headers.clone()); if let Some(context_provider) = &self.namespace_context_provider { @@ -1425,7 +1623,7 @@ impl Database for RemoteDatabase { let mut properties = HashMap::new(); properties.insert("uri".to_string(), self.client.host().to_string()); - properties.insert("delimiter".to_string(), self.client.id_delimiter.clone()); + properties.insert("delimiter".to_string(), ID_DELIMITER.to_string()); for (key, value) in &self.namespace_headers { properties.insert(format!("header.{}", key), value.clone()); } @@ -2208,7 +2406,11 @@ mod tests { http::Response::builder() .status(200) - .body(r#"{"tables": ["ns1$ns2$table1", "ns1$ns2$table2"]}"#) + // A namespace listing names the tables in that namespace; the + // namespace is the route, not part of each name. The client + // joins the two itself to build each table's identifier, so a + // listing that repeated the namespace would be joined twice. + .body(r#"{"tables": ["table1", "table2"]}"#) .unwrap() }); let names = conn @@ -2217,7 +2419,7 @@ mod tests { .execute() .await .unwrap(); - assert_eq!(names, vec!["ns1$ns2$table1", "ns1$ns2$table2"]); + assert_eq!(names, vec!["table1", "table2"]); } #[tokio::test] @@ -3124,6 +3326,354 @@ mod tests { assert_eq!(batches[0].schema(), schema); } + /// A component that is not a legal Secret component never reaches a + /// transport. Before the identifier was checked here, each of these decided + /// the route instead of the name: `a/b` and `../jobs` left `/v1/secret/` + /// entirely, carrying a create body that holds a credential, and `a$b` read + /// as the namespace `a` and the name `b`. + #[tokio::test] + async fn test_an_illegal_component_never_reaches_the_transport() { + use std::sync::{Arc, Mutex}; + for name in [ + "../jobs", + "a/b", + "a$b", + "with space", + "q?x", + "a#b", + "a%2Fb", + "", + ] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .create_secret(name, "sk-live-0001", &[]) + .await + .expect_err("an illegal component must be refused"); + assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); + assert!( + error.to_string().contains("Secret name"), + "{name:?}: {error}" + ); + } + } + + /// `.` and `..` pass the character set and still cannot address anything: + /// URL parsing resolves them as relative segments, and after + /// percent-decoding, so no spelling of either survives. + #[tokio::test] + async fn test_a_relative_segment_component_is_refused() { + use std::sync::{Arc, Mutex}; + // The percent-encoded spellings are caught a step earlier, by the + // character set: `%` is not a character a Secret component may hold. + // They reach the relative-segment rule only where there is no charset + // to catch them first -- see the Function case below. + for component in [".", ".."] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder() + .status(200) + .body(r#"{"secrets":[]}"#) + .unwrap() + }); + let by_name = conn + .drop_secret(component, &[]) + .await + .expect_err("a dot-only name must be refused"); + assert!( + by_name.to_string().contains("relative path segments"), + "{by_name}" + ); + + let by_segment = conn + .list_secrets(&[component.to_string()]) + .await + .expect_err("a dot-only namespace segment must be refused"); + assert!( + by_segment.to_string().contains("relative path segments"), + "{by_segment}" + ); + assert!( + !*reached.lock().unwrap(), + "{component:?} reached the transport" + ); + } + } + + /// An identifier the service accepts is untouched by the encoding, so the + /// route reads like the table and Function routes beside it. + #[tokio::test] + async fn test_an_admissible_identifier_is_not_encoded() { + use std::sync::{Arc, Mutex}; + let seen = Arc::new(Mutex::new(String::new())); + let captured = seen.clone(); + let conn = Connection::new_with_handler(move |request| { + *captured.lock().unwrap() = request.url().path().to_string(); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret( + "openai-prod.v1", + &["prod".to_string(), "vision_2".to_string()], + ) + .await + .unwrap(); + assert_eq!( + *seen.lock().unwrap(), + "/v1/secret/prod$vision_2$openai-prod.v1/drop" + ); + } + + /// A table name is joined into the route the same way a Secret's is, so the + /// same two failures are reachable: a dot-only name leaves the table route + /// entirely, and a name holding the delimiter is indistinguishable from a + /// namespace boundary. + #[tokio::test] + async fn test_a_table_name_cannot_choose_its_own_route() { + use std::sync::{Arc, Mutex}; + for name in ["..", ".", "../jobs", "a/b", "a$b"] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .drop_table(name, &[]) + .await + .expect_err("an unaddressable table name must be refused"); + assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); + assert!(!error.to_string().is_empty(), "{name:?}"); + } + } + + /// The namespace half of the same join. + #[tokio::test] + async fn test_a_namespace_segment_cannot_choose_its_own_route() { + use std::sync::{Arc, Mutex}; + for segment in ["..", "a$b", ""] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .drop_table("t", &[segment.to_string()]) + .await + .expect_err("an unaddressable namespace segment must be refused"); + assert!( + !*reached.lock().unwrap(), + "{segment:?} reached the transport" + ); + assert!(!error.to_string().is_empty(), "{segment:?}"); + } + } + + /// A segment outside the table charset still addresses one segment: the + /// service decides whether it may exist, and percent-encoding is what keeps + /// the question reaching the right route. A catalog database is named this + /// way. + #[tokio::test] + async fn test_a_namespace_segment_outside_the_charset_is_encoded_not_refused() { + use std::sync::{Arc, Mutex}; + let seen = Arc::new(Mutex::new(String::new())); + let path = seen.clone(); + let conn = Connection::new_with_handler(move |request| { + *path.lock().unwrap() = request.url().path().to_string(); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_table("t", &["team/search".to_string()]) + .await + .unwrap(); + assert_eq!(*seen.lock().unwrap(), "/v1/table/team%2Fsearch$t/drop/"); + } + + /// A Function name is percent-encoded, which covers everything but the + /// relative segment: `..` is unreserved, so it survives encoding and is + /// then resolved away, posting a registration body to `/v1/create`. + #[tokio::test] + async fn test_a_relative_segment_function_name_is_refused() { + use std::sync::{Arc, Mutex}; + for name in [".", "..", "%2E%2E"] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .drop_function(name, "fv_1") + .await + .expect_err("a dot-only Function name must be refused"); + assert!( + error.to_string().contains("relative path segments"), + "{error}" + ); + assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); + } + } + + /// Only `.` and `..` are relative segments. `...` and longer runs are + /// ordinary and address perfectly well, so refusing them would make an + /// object that works today stop working on upgrade. This pins that. + #[tokio::test] + async fn test_a_longer_run_of_periods_is_an_ordinary_name() { + use std::sync::{Arc, Mutex}; + for name in ["...", "....", "a.", ".a", "a..b"] { + let seen = Arc::new(Mutex::new(String::new())); + let captured = seen.clone(); + let conn = Connection::new_with_handler(move |request| { + *captured.lock().unwrap() = request.url().path().to_string(); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret(name, &[]) + .await + .unwrap_or_else(|error| panic!("{name:?} must remain addressable: {error}")); + assert_eq!( + *seen.lock().unwrap(), + format!("/v1/secret/{name}/drop"), + "{name:?} did not reach its own route" + ); + } + } + + #[tokio::test] + async fn test_create_and_alter_secret_send_the_value_in_the_request_body() { + for (route, call) in [ + ("/v1/secret/openai-prod/create", true), + ("/v1/secret/openai-prod/alter", false), + ] { + let conn = Connection::new_with_handler(move |request| { + assert_eq!(request.method(), &reqwest::Method::POST); + assert_eq!(request.url().path(), route); + // Never a path segment or query parameter, which is what keeps + // it out of access logs and proxy traces. + assert!(request.url().query().is_none(), "{:?}", request.url()); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + // The name is the path identifier, so the body is the value alone. + assert_eq!(body, serde_json::json!({ "value": "sk-live-0001" })); + http::Response::builder().status(200).body("{}").unwrap() + }); + if call { + conn.create_secret("openai-prod", "sk-live-0001", &[]) + .await + .unwrap(); + } else { + conn.alter_secret("openai-prod", "sk-live-0001", &[]) + .await + .unwrap(); + } + } + } + + #[tokio::test] + async fn test_list_secrets_walks_pages_and_returns_names_only() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/$/secret/list"); + let page = request + .url() + .query_pairs() + .find(|(key, _)| key == "page_token") + .map(|(_, value)| value.into_owned()); + let body = match page.as_deref() { + None => r#"{"secrets":[{"name":"openai-prod"}],"page_token":"p2"}"#, + Some("p2") => r#"{"secrets":[{"name":"hf-prod"}]}"#, + Some(other) => panic!("unexpected page token: {other}"), + }; + http::Response::builder().status(200).body(body).unwrap() + }); + assert_eq!( + conn.list_secrets(&[]).await.unwrap(), + vec!["openai-prod".to_string(), "hf-prod".to_string()] + ); + } + + /// A server that keeps handing back the same token would otherwise spin + /// forever. + #[tokio::test] + async fn test_list_secrets_rejects_a_repeated_page_token() { + let conn = Connection::new_with_handler(|_| { + http::Response::builder() + .status(200) + .body(r#"{"secrets":[{"name":"openai-prod"}],"page_token":"same"}"#) + .unwrap() + }); + let error = conn.list_secrets(&[]).await.unwrap_err(); + assert!( + error.to_string().contains("repeated a page_token"), + "{error}" + ); + } + + #[tokio::test] + async fn test_drop_and_describe_address_the_secret_in_the_path() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/secret/openai-prod/drop"); + // Nothing is left to say once the path names the Secret. + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret("openai-prod", &[]).await.unwrap(); + + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/secret/openai-prod/describe"); + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder() + .status(200) + .body(r#"{"name":"openai-prod","created_at_millis":1,"updated_at_millis":2}"#) + .unwrap() + }); + let info = conn.describe_secret("openai-prod", &[]).await.unwrap(); + assert_eq!(info.name, "openai-prod"); + } + + /// The namespace is part of the identifier the path addresses, joined with + /// the client's configured delimiter the way every other object's is. A + /// root Secret is therefore addressed by its bare name, and a namespaced + /// one by the joined path -- there is no body field either way. + #[tokio::test] + async fn test_a_namespace_path_is_addressed_in_the_path() { + let conn = Connection::new_with_handler(|request| { + assert_eq!( + request.url().path(), + "/v1/secret/prod$vision$openai-prod/drop" + ); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret("openai-prod", &["prod".to_string(), "vision".to_string()]) + .await + .unwrap(); + + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/secret/openai-prod/drop"); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret("openai-prod", &[]).await.unwrap(); + + // Listing is namespace-scoped, so the namespace is the whole identifier. + let conn = Connection::new_with_handler(|request| { + assert_eq!( + request.url().path(), + "/v1/namespace/prod$vision/secret/list" + ); + http::Response::builder() + .status(200) + .body(r#"{"secrets":[]}"#) + .unwrap() + }); + conn.list_secrets(&["prod".to_string(), "vision".to_string()]) + .await + .unwrap(); + } + #[tokio::test] async fn test_create_function_async_sends_canonical_request_and_decodes_typed_job() { const REQUEST: &str = include_str!( diff --git a/rust/lancedb/src/secrets.rs b/rust/lancedb/src/secrets.rs new file mode 100644 index 000000000..c980e51ee --- /dev/null +++ b/rust/lancedb/src/secrets.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Named Secrets: database-scoped credentials a Function binds by name. +//! +//! Nothing here holds a credential. The verbs live on +//! [`crate::connection::Connection`], and none of them returns a value -- by +//! construction rather than by policy, so there is no code path that could. +//! +//! What a Secret is, how one is named, and how a Function binds one all live +//! here; [`crate::function`] holds the bindings a FunctionVersion records, the +//! way Sophon's Secret catalog and Function catalog divide the same two. + +use serde::de; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; + +/// What a database records about a Secret. Never its value. +/// +/// Returned by [`crate::connection::Connection::describe_secret`]. There is no +/// field for the credential and no method that could produce one. +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +pub struct SecretInfo { + /// The Secret's database-scoped name. + pub name: String, + /// When the Secret was created, in milliseconds since the Unix epoch. + pub created_at_millis: i64, + /// When the Secret's value was last rotated, in milliseconds since the Unix + /// epoch. + /// + /// This is the only observable that a rotation landed: no API returns a + /// credential, so a caller confirms `alter_secret` took effect by watching + /// this move. + pub updated_at_millis: i64, +} + +/// Where a Secret lives, carried as its parts rather than as one string. +/// +/// Nothing here is parsed, so nothing can parse two ways. A joined id would +/// instead need a delimiter excluded from every name and segment, agreed on by +/// both sides. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SecretReference { + pub name: String, + /// The namespace holding the Secret. Empty is the root, and is absent from + /// the wire rather than sent empty: a binding states a namespace only when + /// it has one. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub namespace_path: Vec, +} + +impl SecretReference { + /// A Secret in the root namespace. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + namespace_path: Vec::new(), + } + } + + /// A Secret in `namespace_path`. + pub fn in_namespace(name: impl Into, namespace_path: Vec) -> Self { + Self { + name: name.into(), + namespace_path, + } + } +} + +/// How a Secret reaches the Function that binds it. +/// +/// One list rather than a field per delivery mode, so a mode added later is a +/// variant and the per-Function rules -- how many Secrets a Function may bind, +/// which ones it needs -- stay answerable from one place. +/// +/// A mode this client does not know decodes rather than failing the whole +/// FunctionVersion, as [`PythonRuntimeSpec`] does for an unknown runtime. That +/// takes both halves: [`SecretBinding::Unrecognized`] gives the wire somewhere +/// to land, and `#[non_exhaustive]` denies callers an exhaustive match, so a +/// later mode arrives as a case they already had to handle. Its payload is +/// dropped -- the client does not proxy catalog values. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[non_exhaustive] +pub enum SecretBinding { + /// Delivered as an environment variable, which the UDF's library already + /// reads. The variable is the delivery target; the Secret is what fills it. + Env { + variable: String, + /// Named `secret_ref` rather than `secret` because a Job payload is + /// scanned server-side for credential-shaped keys, and a key called + /// `secret` trips that guard whatever it actually holds. + secret_ref: SecretReference, + }, + /// A binding kind introduced by a newer server. + Unrecognized { kind: String }, +} + +impl SecretBinding { + /// The wire discriminator reported by Sophon. + pub fn kind(&self) -> &str { + match self { + Self::Env { .. } => "env", + Self::Unrecognized { kind } => kind, + } + } + + /// The environment variable this binding fills, or `None` for a kind that + /// does not deliver through one. + pub fn variable(&self) -> Option<&str> { + match self { + Self::Env { variable, .. } => Some(variable), + Self::Unrecognized { .. } => None, + } + } + + /// The Secret bound, or `None` for a kind this client cannot read. + pub fn secret(&self) -> Option<&SecretReference> { + match self { + Self::Env { secret_ref, .. } => Some(secret_ref), + Self::Unrecognized { .. } => None, + } + } +} + +#[derive(Deserialize)] +struct EnvSecretBindingWire { + variable: String, + secret_ref: SecretReference, +} + +impl<'de> Deserialize<'de> for SecretBinding { + fn deserialize>(deserializer: D) -> std::result::Result { + let value = Value::deserialize(deserializer)?; + let kind = value + .get("kind") + .ok_or_else(|| de::Error::missing_field("kind"))? + .as_str() + .ok_or_else(|| de::Error::custom("secret binding kind must be a string"))? + .to_string(); + match kind.as_str() { + "env" => { + let wire: EnvSecretBindingWire = + serde_json::from_value(value).map_err(de::Error::custom)?; + Ok(Self::Env { + variable: wire.variable, + secret_ref: wire.secret_ref, + }) + } + _ => Ok(Self::Unrecognized { kind }), + } + } +} + +impl Serialize for SecretBinding { + fn serialize(&self, serializer: S) -> std::result::Result { + #[derive(Serialize)] + struct EnvBindingRef<'a> { + kind: &'static str, + variable: &'a str, + secret_ref: &'a SecretReference, + } + + #[derive(Serialize)] + struct UnrecognizedBindingRef<'a> { + kind: &'a str, + } + + match self { + Self::Env { + variable, + secret_ref, + } => EnvBindingRef { + kind: "env", + variable, + secret_ref, + } + .serialize(serializer), + Self::Unrecognized { kind } => UnrecognizedBindingRef { kind }.serialize(serializer), + } + } +} diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index a66e758f7..f80cc876a 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -19,10 +19,15 @@ use std::pin::Pin; use crate::error::{Error, Result}; use datafusion_physical_plan::SendableRecordBatchStream; -static TABLE_NAME_REGEX: std::sync::LazyLock = - std::sync::LazyLock::new(|| regex::Regex::new(r"^[a-zA-Z0-9_\-\.]+$").unwrap()); -static NAMESPACE_NAME_REGEX: std::sync::LazyLock = - std::sync::LazyLock::new(|| regex::Regex::new(r"^[a-zA-Z0-9_\-\.]+$").unwrap()); +/// The characters any object name may contain: a table, a namespace segment, a +/// Secret, a materialized view. +/// +/// No positional rule on top of it -- a name may begin with `_`, `-` or `.`, +/// as LanceDB namespaces already do. `.` and `..` are excluded separately, by +/// [`reject_relative_segment`]: that is a property of where a name sits in a +/// URL, not of the name. Length is the service's to bound. +static OBJECT_NAME_REGEX: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r"^[A-Za-z0-9_.\-]+$").unwrap()); pub trait PatchStoreParam { fn patch_with_store_wrapper( @@ -81,66 +86,94 @@ impl PatchReadParam for ReadParams { } } -/// Validate table name. -pub fn validate_table_name(name: &str) -> Result<()> { - if name.is_empty() { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: "Table names cannot be empty strings".to_string(), - }); - } - if name == "." { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: "Table name cannot be a single dot.".to_string(), - }); - } - if name == ".." { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: "Table name cannot be two dots.".to_string(), - }); - } - if !TABLE_NAME_REGEX.is_match(name) { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: - "Table names can only contain alphanumeric characters, underscores, hyphens, and periods" - .to_string(), +/// The reason `.` and `..` are refused wherever a name becomes a path segment. +const RELATIVE_SEGMENT_REASON: &str = + "'.' and '..' are read as relative path segments and cannot address an object"; + +/// Whether URL parsing would resolve this component away rather than keep it. +/// +/// Exactly `.` and `..`, and their percent-encoded spellings -- resolution +/// happens after decoding, so `%2E%2E` collapses as surely as `..` does, and +/// `drop_table("..")` would reach `/v1/drop/`. No wider than that: `...` is an +/// ordinary segment that addresses fine. +fn is_relative_segment(value: &str) -> bool { + let decoded = value.replace("%2e", ".").replace("%2E", "."); + decoded == "." || decoded == ".." +} + +/// Refuse a path component that URL parsing resolves as a relative segment. +/// +/// Reachable on its own for an identifier with no other validator: a Function +/// name has no client-side grammar, so this is the only rule that applies. +pub(crate) fn reject_relative_segment(what: &str, value: &str) -> Result<()> { + if is_relative_segment(value) { + return Err(Error::InvalidInput { + message: format!("invalid {what} '{value}': {RELATIVE_SEGMENT_REASON}"), }); } Ok(()) } -/// Validate a namespace name component +/// Every rule an object name obeys: non-empty, inside [`OBJECT_NAME_REGEX`], +/// and addressable as a path segment. /// -/// Namespace names must: -/// - Not be empty -/// - Only contain alphanumeric characters, underscores, hyphens, and periods -/// -/// # Arguments -/// * `name` - A single namespace component (not the full path) -/// -/// # Returns -/// * `Ok(())` if the namespace name is valid -/// * `Err(Error)` if the namespace name is invalid -pub fn validate_namespace_name(name: &str) -> Result<()> { +/// Returns the reason rather than an [`Error`], because the error type is each +/// API's own -- a table reports [`Error::InvalidTableName`], the rest +/// [`Error::InvalidInput`]. Sharing the rules but not the error keeps a table, +/// a namespace segment and a Secret from drifting apart. +fn check_object_name(name: &str) -> std::result::Result<(), &'static str> { if name.is_empty() { - return Err(Error::InvalidInput { - message: "Namespace names cannot be empty strings".to_string(), - }); + return Err("it must not be empty"); } - if !NAMESPACE_NAME_REGEX.is_match(name) { - return Err(Error::InvalidInput { - message: format!( - "Invalid namespace name '{}': Namespace names can only contain alphanumeric characters, underscores, hyphens, and periods", - name - ), - }); + if !OBJECT_NAME_REGEX.is_match(name) { + return Err( + "it may contain only alphanumeric characters, underscores, hyphens and periods", + ); + } + if is_relative_segment(name) { + return Err(RELATIVE_SEGMENT_REASON); } Ok(()) } +/// Validate a table name. +pub fn validate_table_name(name: &str) -> Result<()> { + check_object_name(name).map_err(|reason| Error::InvalidTableName { + name: name.to_string(), + reason: reason.to_string(), + }) +} + +/// Validate one component of a namespace path -- a single segment, not the +/// whole path. [`validate_namespace`] covers a path. +pub fn validate_namespace_name(name: &str) -> Result<()> { + check_object_name(name).map_err(|reason| Error::InvalidInput { + message: format!("invalid namespace name '{name}': {reason}"), + }) +} + +/// Validate one component of a Secret identifier: a Secret name, or one segment +/// of the namespace path holding it. +/// +/// The join decides identity, and the service only sees what the split +/// produced. `"a$b"` is not a name the service accepts, but joined and split it +/// reads as the namespace `a` and the name `b` -- a different Secret that may +/// already exist. This is not a second opinion on the name; it is what lets the +/// service have one. +pub fn validate_secret_component(what: &str, value: &str) -> Result<()> { + check_object_name(value).map_err(|reason| Error::InvalidInput { + message: format!("invalid {what} '{value}': {reason}"), + }) +} + +/// Validate a Secret name and every segment of the namespace path holding it. +pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Result<()> { + for segment in namespace_path { + validate_secret_component("Secret namespace path segment", segment)?; + } + validate_secret_component("Secret name", name) +} + /// Validate all components of a namespace /// /// Iterates through all namespace components and validates each one. diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index b70ac7f0a..3971a2471 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use lancedb::function::{ FunctionApplication, FunctionBinding, FunctionVersion, RefreshColumnResult, }; +use lancedb::secrets::{SecretBinding, SecretReference}; use serde_json::Value; fn fixture(name: &str) -> String { @@ -20,6 +21,26 @@ fn job_result(name: &str) -> Value { serde_json::from_str::(&fixture(name)).expect("remote Job fixture")["result"].clone() } +/// No client value models a resolved credential, at any nesting depth. +fn assert_no_secret_values(value: &Value) { + match value { + Value::Object(values) => { + for (key, value) in values { + assert!( + !matches!( + key.as_str(), + "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets" + ), + "client canonical value must not model resolved secret material" + ); + assert_no_secret_values(value); + } + } + Value::Array(values) => values.iter().for_each(assert_no_secret_values), + _ => {} + } +} + #[test] fn function_version_job_result_matches_shared_canonical_golden() { let result = job_result("remote_function_job.json"); @@ -28,6 +49,13 @@ fn function_version_job_result_matches_shared_canonical_golden() { assert_eq!(version.name(), "embed"); assert_eq!(version.version(), "1"); assert_ne!(version.image().manifest_digest, version.version()); + assert_eq!( + version.secret_bindings(), + [SecretBinding::Env { + variable: "HF_TOKEN".to_string(), + secret_ref: SecretReference::new("hf-prod"), + }] + ); assert_eq!( version.to_canonical_json().expect("canonical JSON"), fixture("remote_function_version.canonical.json").trim() @@ -150,3 +178,75 @@ fn floating_point_application_literals_are_rejected_consistently() { .contains("floating-point Function literals") ); } + +#[test] +fn canonical_client_values_carry_bindings_and_no_credentials() { + let result = job_result("remote_function_job.json"); + let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); + let canonical: Value = serde_json::from_str( + &version + .to_canonical_json() + .expect("canonical FunctionVersion"), + ) + .expect("canonical JSON"); + + assert_eq!( + canonical["secret_bindings"], + serde_json::json!([{"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}}]) + ); + assert_no_secret_values(&canonical); +} + +/// A binding kind a newer server introduces must not fail the whole version. +/// +/// This is the cost the union pays for being one field: an unknown variant is +/// a decode error unless it is caught, so it is caught -- and the payload is +/// dropped rather than retained, as `PythonRuntimeSpec` does, because the +/// client does not proxy catalog values. +#[test] +fn an_unknown_binding_kind_is_forward_decodable() { + let mut result = job_result("remote_function_job.json"); + result["secret_bindings"] = serde_json::json!([ + {"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}}, + {"kind": "file", "path": "/run/secrets/tok", "secret_ref": {"name": "hf-prod"}}, + ]); + + let version = FunctionVersion::from_json(&result.to_string()).expect("future binding kind"); + + let kinds = version + .secret_bindings() + .iter() + .map(|binding| binding.kind()) + .collect::>(); + assert_eq!(kinds, ["env", "file"]); + assert_eq!(version.secret_bindings()[1].variable(), None); + assert_eq!(version.secret_bindings()[1].secret(), None); + + // The unknown kind round-trips as its discriminator and nothing more. + let canonical: Value = + serde_json::from_str(&version.to_canonical_json().expect("canonical")).expect("JSON"); + assert_eq!( + canonical["secret_bindings"][1], + serde_json::json!({"kind": "file"}) + ); +} + +/// A Function that binds nothing carries no `secret_bindings` key: absent +/// decodes as an empty list, and an empty list serializes back to absent. +#[test] +fn a_version_without_bindings_omits_the_field_in_both_directions() { + let mut result = job_result("remote_function_job.json"); + result + .as_object_mut() + .expect("Function version object") + .remove("secret_bindings"); + let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); + + assert!(version.secret_bindings().is_empty()); + assert!( + !version + .to_canonical_json() + .expect("canonical FunctionVersion") + .contains("secret_bindings") + ); +} diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs index 65924f394..14ceb511d 100644 --- a/rust/lancedb/tests/first_class_function_slice2.rs +++ b/rust/lancedb/tests/first_class_function_slice2.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use lancedb::Error; use lancedb::function::FunctionRegistrationRequest; +use lancedb::secrets::{SecretBinding, SecretReference}; +use serde_json::Value; fn fixture(name: &str) -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -14,6 +16,26 @@ fn fixture(name: &str) -> String { fs::read_to_string(path).expect("fixture must be readable") } +/// A registration request never models a resolved credential, at any depth. +fn assert_no_secret_values(value: &Value) { + match value { + Value::Object(values) => { + for (key, value) in values { + assert!( + !matches!( + key.as_str(), + "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets" + ), + "registration requests must not model resolved secret material" + ); + assert_no_secret_values(value); + } + } + Value::Array(values) => values.iter().for_each(assert_no_secret_values), + _ => {} + } +} + #[test] fn registration_request_matches_shared_canonical_golden() { let request = FunctionRegistrationRequest::from_json(&fixture( @@ -22,10 +44,45 @@ fn registration_request_matches_shared_canonical_golden() { .expect("registration request"); assert_eq!(request.name, "normalize_score"); assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch"); + // The unchanged path: a Function that binds nothing serializes today's + // bytes, with no `secret_bindings` key at all. + assert!(request.secret_bindings.is_empty()); assert_eq!( request.to_canonical_json().expect("canonical request"), fixture("remote_function_registration_request.canonical.json").trim() ); + + let value: Value = + serde_json::from_str(&request.to_canonical_json().expect("canonical request")) + .expect("request JSON"); + assert_no_secret_values(&value); +} + +/// The same shared golden as the Python suite builds from `@udf(secrets=...)` +/// plus `bind_secrets`, so both clients agree byte for byte on a bound request. +#[test] +fn secret_bound_registration_request_matches_shared_canonical_golden() { + let request = FunctionRegistrationRequest::from_json(&fixture( + "remote_function_secret_registration_request.json", + )) + .expect("registration request"); + assert_eq!(request.name, "analyze_caption"); + assert_eq!( + request.secret_bindings, + [SecretBinding::Env { + variable: "OPENAI_API_KEY".to_string(), + secret_ref: SecretReference::new("openai-prod"), + }] + ); + assert_eq!( + request.to_canonical_json().expect("canonical request"), + fixture("remote_function_secret_registration_request.canonical.json").trim() + ); + + let value: Value = + serde_json::from_str(&request.to_canonical_json().expect("canonical request")) + .expect("request JSON"); + assert_no_secret_values(&value); } #[tokio::test] diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json index 3381fd266..c6bbd83e3 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json @@ -58,7 +58,16 @@ } }, "source": false - } + }, + "secret_bindings": [ + { + "kind": "env", + "variable": "HF_TOKEN", + "secret_ref": { + "name": "hf-prod" + } + } + ] }, "future_job": { "trace_id": "trace-1" diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json new file mode 100644 index 000000000..5cce05e9b --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json @@ -0,0 +1 @@ +{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_bindings":[{"kind":"env","secret_ref":{"name":"openai-prod"},"variable":"OPENAI_API_KEY"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json new file mode 100644 index 000000000..6edea0a91 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json @@ -0,0 +1,52 @@ +{ + "artifact": { + "adapter": { + "kind": "scalar_to_arrow_batch", + "version": 1 + }, + "content": { + "data": "ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK", + "encoding": "base64" + }, + "digest": "sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077", + "entrypoint": "analyze_caption", + "kind": "python_callable" + }, + "name": "analyze_caption", + "runtime": { + "env": { + "MODE": "test" + }, + "environment": { + "kind": "pip", + "packages": [ + "openai==3.7.0" + ] + }, + "kind": "python", + "python_version": "3.12" + }, + "signature": { + "inputs": [ + { + "arrow_type": "utf8", + "name": "caption", + "nullable": false + } + ], + "output": { + "arrow_type": "utf8", + "kind": "scalar", + "nullable": false + } + }, + "secret_bindings": [ + { + "kind": "env", + "variable": "OPENAI_API_KEY", + "secret_ref": { + "name": "openai-prod" + } + } + ] +} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json index 3f0c11973..63f3c824e 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json @@ -1 +1 @@ -{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"} +{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","secret_bindings":[{"kind":"env","secret_ref":{"name":"hf-prod"},"variable":"HF_TOKEN"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"}