mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-10 23:32:35 +00:00
feat(secrets): named Secrets and EnvVarSecret bindings
Adds the client half of database-scoped named Secrets: a Secret is a name and an opaque value stored by the service, and a Function binds one to the environment variable its library already reads. - `db.create_secret` / `alter_secret` / `list_secrets` / `describe_secret` / `drop_secret` on sync, async and remote connections, with the pyo3 binding and the Rust client behind them. There is no read API by construction rather than by policy: no code path returns a stored credential, and `describe_secret` answers with metadata only. - `EnvVarSecret(secret=..., env_variable=...)` pairs a Secret with the variable it arrives in. It is a pure local constructor -- it contacts no server, so it cannot fail on a Secret that does not exist -- and it exists so a bare string in that position, which would be a credential, is a TypeError rather than a plausible-looking mistake that reads identically in a diff. - `create_function(..., secrets=[...])` carries the bindings as `secret_bindings`, a map from variable name to Secret name. The value never travels: it is resolved by the service when the Function runs, which is what lets a rotation reach columns pinned to an older FunctionVersion. The UDF body is unchanged and stays portable -- it reads `OPENAI_API_KEY` the way it always did, and the binding is what puts a value there. Squashed: the original three commits were a first design plus a rewrite of it, so their sequence describes an interface that no longer exists. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XE1UwYKsgbb3USBfkqCE6v
This commit is contained in:
co-authored by
Claude Opus 5
parent
0111a72dc3
commit
7b29fb2f51
@@ -37,6 +37,8 @@ from .functions import (
|
||||
UdfDefinition as UdfDefinition,
|
||||
udf as udf,
|
||||
)
|
||||
from .secrets import EnvVarSecret as EnvVarSecret
|
||||
from .secrets import SecretInfo as SecretInfo
|
||||
from .materialized_view import (
|
||||
AsyncMaterializedView,
|
||||
MaterializedView,
|
||||
|
||||
@@ -153,6 +153,11 @@ class Connection(object):
|
||||
async def get_function(self, name: str, version: str) -> str: ...
|
||||
async def list_functions(self) -> List[str]: ...
|
||||
async def drop_function(self, name: str, version: str) -> bool: ...
|
||||
async def create_secret(self, name: str, value: str) -> None: ...
|
||||
async def alter_secret(self, name: str, value: str) -> None: ...
|
||||
async def list_secrets(self) -> List[str]: ...
|
||||
async def drop_secret(self, name: str) -> None: ...
|
||||
async def describe_secret(self, name: str) -> Dict[str, str]: ...
|
||||
async def list_jobs(self) -> List[JobInfo]: ...
|
||||
async def cancel_job(self, job_id: str) -> bool: ...
|
||||
async def execute_query_async(
|
||||
|
||||
+161
-11
@@ -17,6 +17,7 @@ from typing import (
|
||||
List,
|
||||
Literal,
|
||||
Optional,
|
||||
Sequence,
|
||||
Union,
|
||||
)
|
||||
from uuid import UUID
|
||||
@@ -57,6 +58,7 @@ from .materialized_view import (
|
||||
SelectArg,
|
||||
normalize_select,
|
||||
)
|
||||
from .secrets import EnvVarSecret, SecretInfo, validate_secret_name
|
||||
from .table import (
|
||||
AsyncTable,
|
||||
LanceTable,
|
||||
@@ -692,15 +694,47 @@ class DBConnection(EnforceOverrides):
|
||||
"""
|
||||
raise NotImplementedError("serialize is not supported for this connection type")
|
||||
|
||||
def create_function(self, definition: UdfDefinition) -> FunctionVersion:
|
||||
def create_function(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
) -> FunctionVersion:
|
||||
"""Register a scalar Python UDF and wait for its immutable version.
|
||||
|
||||
This is the blocking counterpart of :meth:`create_function_async`.
|
||||
Local connections raise ``NotImplementedError``.
|
||||
"""
|
||||
return self.create_function_async(definition).wait()
|
||||
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
Parameters
|
||||
----------
|
||||
definition : UdfDefinition
|
||||
A callable decorated with [udf][lancedb.udf].
|
||||
secrets : sequence of EnvVarSecret, optional
|
||||
One [EnvVarSecret][lancedb.secrets.EnvVarSecret] per credential the
|
||||
Function needs, each naming a Secret and the environment variable
|
||||
its value arrives in. The Function's source is unchanged by this;
|
||||
it reads the variable the way it already did.
|
||||
|
||||
Examples
|
||||
--------
|
||||
```python
|
||||
db.create_secret("openai-prod", os.environ["OPENAI_API_KEY"])
|
||||
db.create_function(
|
||||
analyze_caption,
|
||||
secrets=[
|
||||
EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
|
||||
],
|
||||
)
|
||||
```
|
||||
"""
|
||||
return self.create_function_async(definition, secrets=secrets).wait()
|
||||
|
||||
def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
"""Register a scalar Python UDF through the remote Function catalog.
|
||||
|
||||
Submission returns a typed job. The immutable Function version becomes
|
||||
@@ -745,6 +779,62 @@ class DBConnection(EnforceOverrides):
|
||||
"Function catalog operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def create_secret(self, name: str, value: str) -> None:
|
||||
"""Create a named Secret in this database.
|
||||
|
||||
Fails if the name is taken, so a create never silently becomes a
|
||||
rotation. Nothing reads the value back: it is bound to a Function by
|
||||
name and resolved by the service when that Function runs. Local
|
||||
connections raise ``NotImplementedError``.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Secret operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def alter_secret(self, name: str, value: str) -> None:
|
||||
"""Replace the credential behind an existing Secret.
|
||||
|
||||
Fails if it does not exist. Every Function bound to the Secret uses the
|
||||
new value from its next job, and no new Function version is created --
|
||||
which is how a rotation reaches columns pinned to a version registered
|
||||
before it. Local connections raise ``NotImplementedError``.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Secret operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def list_secrets(self) -> List[str]:
|
||||
"""The names of every Secret in this database.
|
||||
|
||||
Names only. No method returns a stored credential, by construction
|
||||
rather than by policy. Local connections raise ``NotImplementedError``.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Secret operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def drop_secret(self, name: str) -> None:
|
||||
"""Drop a Secret.
|
||||
|
||||
Functions bound to it fail at their next job, naming the Secret; that
|
||||
is the revocation path. The name becomes free to reuse, and a new
|
||||
Secret under it is picked up by everything still bound to that name.
|
||||
Local connections raise ``NotImplementedError``.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Secret operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def describe_secret(self, name: str) -> SecretInfo:
|
||||
"""What this database records about a Secret: name and timestamps.
|
||||
|
||||
Never the value -- there is no code path that could return one. Local
|
||||
connections raise ``NotImplementedError``.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"Secret operations are not supported for this connection type"
|
||||
)
|
||||
|
||||
def open_job(self, job_id: str) -> Job:
|
||||
"""Open a server-side job by id, returning a handle with its record
|
||||
already populated.
|
||||
@@ -1457,8 +1547,13 @@ class LanceDBConnection(DBConnection):
|
||||
return Job(LOOP.run(self._conn.open_job(job_id)))
|
||||
|
||||
@override
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(self._conn.create_function_async(definition))
|
||||
def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
|
||||
return Job(job)
|
||||
|
||||
@override
|
||||
@@ -1473,6 +1568,26 @@ class LanceDBConnection(DBConnection):
|
||||
def drop_function(self, name: str, *, version: str) -> bool:
|
||||
return LOOP.run(self._conn.drop_function(name, version=version))
|
||||
|
||||
@override
|
||||
def create_secret(self, name: str, value: str) -> None:
|
||||
LOOP.run(self._conn.create_secret(name, value))
|
||||
|
||||
@override
|
||||
def alter_secret(self, name: str, value: str) -> None:
|
||||
LOOP.run(self._conn.alter_secret(name, value))
|
||||
|
||||
@override
|
||||
def list_secrets(self) -> List[str]:
|
||||
return LOOP.run(self._conn.list_secrets())
|
||||
|
||||
@override
|
||||
def drop_secret(self, name: str) -> None:
|
||||
LOOP.run(self._conn.drop_secret(name))
|
||||
|
||||
@override
|
||||
def describe_secret(self, name: str) -> SecretInfo:
|
||||
return LOOP.run(self._conn.describe_secret(name))
|
||||
|
||||
@override
|
||||
def list_jobs(self) -> List[JobInfo]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
@@ -2265,18 +2380,23 @@ class AsyncConnection(object):
|
||||
return AsyncJob(await self._inner.open_job(job_id))
|
||||
|
||||
async def create_function_async(
|
||||
self, definition: UdfDefinition
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
) -> AsyncJob[FunctionVersion]:
|
||||
"""Register a scalar Python UDF through the remote Function catalog.
|
||||
|
||||
The returned typed job resolves to the immutable Function version.
|
||||
Local connections raise ``NotImplementedError``.
|
||||
``secrets`` is a sequence of
|
||||
[EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and
|
||||
the environment variable its value arrives in. Local connections raise
|
||||
``NotImplementedError``.
|
||||
"""
|
||||
if not isinstance(definition, UdfDefinition):
|
||||
raise TypeError("create_function_async requires a @udf definition")
|
||||
inner = await self._inner.create_function_async(
|
||||
definition.registration_request.to_canonical_json()
|
||||
)
|
||||
request = definition.bind_secrets(secrets)
|
||||
inner = await self._inner.create_function_async(request.to_canonical_json())
|
||||
return _typed_job(inner, FunctionVersion.from_json)
|
||||
|
||||
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
|
||||
@@ -2298,6 +2418,36 @@ class AsyncConnection(object):
|
||||
"""Drop one exact immutable Function version from the remote catalog."""
|
||||
return await self._inner.drop_function(name, version)
|
||||
|
||||
async def create_secret(self, name: str, value: str) -> None:
|
||||
"""Create a named Secret in this database.
|
||||
|
||||
Fails if the name is taken, so a create never silently becomes a
|
||||
rotation. Nothing reads the value back.
|
||||
"""
|
||||
await self._inner.create_secret(validate_secret_name(name), value)
|
||||
|
||||
async def alter_secret(self, name: str, value: str) -> None:
|
||||
"""Replace the credential behind an existing Secret.
|
||||
|
||||
Fails if it does not exist. Bound Functions use the new value from
|
||||
their next job, with no new Function version.
|
||||
"""
|
||||
await self._inner.alter_secret(validate_secret_name(name), value)
|
||||
|
||||
async def list_secrets(self) -> List[str]:
|
||||
"""The names of every Secret in this database. Names only."""
|
||||
return await self._inner.list_secrets()
|
||||
|
||||
async def drop_secret(self, name: str) -> None:
|
||||
"""Drop a Secret. Bound Functions fail at their next job."""
|
||||
await self._inner.drop_secret(validate_secret_name(name))
|
||||
|
||||
async def describe_secret(self, name: str) -> SecretInfo:
|
||||
"""What this database records about a Secret. Never the value."""
|
||||
return SecretInfo.from_json(
|
||||
await self._inner.describe_secret(validate_secret_name(name))
|
||||
)
|
||||
|
||||
async def list_jobs(self) -> List[JobInfo]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
return await self._inner.list_jobs()
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"""Canonical Function values exchanged with LanceDB Enterprise services.
|
||||
|
||||
These immutable models contain client/wire state only. Catalog persistence,
|
||||
environment bake, and execution are owned by Sophon.
|
||||
environment bake, secret resolution, and execution are owned by Sophon.
|
||||
``RefreshColumnResult`` is also the backend-neutral result of a local
|
||||
expression-backed refresh job.
|
||||
"""
|
||||
@@ -25,7 +25,7 @@ import re
|
||||
import sys
|
||||
import textwrap
|
||||
import types
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import date, datetime
|
||||
from typing import (
|
||||
Annotated,
|
||||
@@ -50,6 +50,7 @@ from pydantic import (
|
||||
)
|
||||
|
||||
from .schema import is_blob_v2_field as _is_blob_v2_field
|
||||
from .secrets import EnvVarSecret
|
||||
|
||||
_Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
|
||||
_UInt32 = conint(strict=True, ge=0, le=2**32 - 1)
|
||||
@@ -309,6 +310,7 @@ class FunctionVersion(_RemoteValue):
|
||||
runtime: PythonRuntimeSpec
|
||||
runtime_digest: str
|
||||
environment_digest: str
|
||||
secret_bindings: Mapping[str, str] = {}
|
||||
created_at: str
|
||||
|
||||
def __call__(self, **inputs: Any) -> FunctionApplication:
|
||||
@@ -370,12 +372,18 @@ class FunctionVersion(_RemoteValue):
|
||||
|
||||
|
||||
class FunctionRegistrationRequest(_RemoteValue):
|
||||
"""Stable remote registration envelope produced by :func:`udf`."""
|
||||
"""Stable remote registration envelope produced by :func:`udf`.
|
||||
|
||||
Credential values deliberately have no field here. The only secret-shaped
|
||||
thing a client sends is ``secret_bindings``: the name of a Secret the
|
||||
database already holds, which the remote service resolves at execution.
|
||||
"""
|
||||
|
||||
name: str
|
||||
artifact: FunctionArtifactRequest
|
||||
signature: FunctionSignature
|
||||
runtime: PythonRuntimeSpec
|
||||
secret_bindings: Mapping[str, str] = {}
|
||||
|
||||
|
||||
class FunctionVersionRef(_OpenRemoteValue):
|
||||
@@ -524,6 +532,14 @@ class RefreshColumnResult(_RemoteValue):
|
||||
|
||||
|
||||
_FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$")
|
||||
_DECLARED_SECRET = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
MAX_FUNCTION_SECRET_BINDINGS = 16
|
||||
"""A Function binds at most this many Secrets.
|
||||
|
||||
Each bound Secret is one extra read on the launch path of every fragment, so
|
||||
the count needs a bound for the same reason a credential needs a size limit.
|
||||
"""
|
||||
_FUNCTION_BLOB_V2_TYPE = "blob_v2"
|
||||
_ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name"
|
||||
_BLOB_V2_EXTENSION_NAME = "lance.blob.v2"
|
||||
@@ -1265,9 +1281,61 @@ class UdfDefinition:
|
||||
|
||||
@property
|
||||
def registration_request(self) -> FunctionRegistrationRequest:
|
||||
"""The immutable request sent by ``create_function_async``."""
|
||||
"""The immutable request sent by ``create_function_async``.
|
||||
|
||||
Carries no secret bindings. Binding is a registration-time decision,
|
||||
so a Function bound to Secrets is registered through :meth:`bind_secrets`,
|
||||
which is what ``create_function`` calls.
|
||||
"""
|
||||
return self._request
|
||||
|
||||
def bind_secrets(
|
||||
self, secrets: Optional[Sequence[EnvVarSecret]]
|
||||
) -> FunctionRegistrationRequest:
|
||||
"""The registration request for this definition bound to ``secrets``.
|
||||
|
||||
Binding does not change the Function's source: each
|
||||
[EnvVarSecret][lancedb.secrets.EnvVarSecret] names a Secret and the
|
||||
environment variable its value should arrive in, and the Function reads
|
||||
that variable the way it already did. Whether the named Secrets exist is
|
||||
the server's answer, not this one.
|
||||
"""
|
||||
bindings = () if secrets is None else tuple(secrets)
|
||||
wrong_type = [
|
||||
binding for binding in bindings if not isinstance(binding, EnvVarSecret)
|
||||
]
|
||||
if wrong_type:
|
||||
kinds = sorted({type(binding).__name__ for binding in wrong_type})
|
||||
raise TypeError(
|
||||
f"Function secrets must be EnvVarSecret values, not {kinds!r}; a "
|
||||
"credential value is never sent to this API"
|
||||
)
|
||||
if len(bindings) > MAX_FUNCTION_SECRET_BINDINGS:
|
||||
raise ValueError(
|
||||
f"a Function binds at most {MAX_FUNCTION_SECRET_BINDINGS} secrets, "
|
||||
f"not {len(bindings)}"
|
||||
)
|
||||
variables = [binding.env_variable for binding in bindings]
|
||||
duplicates = sorted({name for name in variables if variables.count(name) > 1})
|
||||
if duplicates:
|
||||
raise ValueError(
|
||||
"a Function binds each environment variable once; duplicated: "
|
||||
f"{duplicates!r}"
|
||||
)
|
||||
# `env` is ordinary configuration carried in the definition, so a name in
|
||||
# both would have a value visible in the Function's record and a value
|
||||
# that is not. Refuse rather than pick.
|
||||
environment = self._request.runtime.env or {}
|
||||
overlap = sorted(set(environment) & set(variables))
|
||||
if overlap:
|
||||
raise ValueError(
|
||||
f"Function env and secret bindings must be disjoint: {overlap!r}"
|
||||
)
|
||||
if not bindings:
|
||||
return self._request
|
||||
resolved = {binding.env_variable: binding.secret for binding in bindings}
|
||||
return self._request._copy(update={"secret_bindings": resolved})
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
return self._function(*args, **kwargs)
|
||||
|
||||
@@ -1332,7 +1400,9 @@ def udf(
|
||||
conda_channels : sequence of str, optional
|
||||
Conda channels in priority order; requires ``conda``.
|
||||
env : mapping of str to str, optional
|
||||
Environment variables included in the Function definition.
|
||||
Environment variables included in the Function definition. Not for
|
||||
credentials -- these are ordinary configuration, stored with the
|
||||
Function and visible wherever it is.
|
||||
python_version : str, optional
|
||||
Remote Python major/minor version. Defaults to the client version.
|
||||
gpu : bool, default False
|
||||
|
||||
@@ -7,7 +7,16 @@ import json
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Sequence,
|
||||
Union,
|
||||
)
|
||||
from urllib.parse import urlparse
|
||||
from uuid import UUID
|
||||
import warnings
|
||||
@@ -29,6 +38,7 @@ from ..job import AsyncJob, Job
|
||||
from ..sql import Query as SqlQuery
|
||||
from ..sql import QueryDescription
|
||||
from ..materialized_view import MaterializedView, SelectArg
|
||||
from ..secrets import EnvVarSecret, SecretInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .._lancedb import JobInfo
|
||||
@@ -746,8 +756,14 @@ class RemoteDBConnection(DBConnection):
|
||||
return Job(LOOP.run(self._conn.open_job(job_id)))
|
||||
|
||||
@override
|
||||
def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]:
|
||||
return Job(LOOP.run(self._conn.create_function_async(definition)))
|
||||
def create_function_async(
|
||||
self,
|
||||
definition: UdfDefinition,
|
||||
*,
|
||||
secrets: Optional[Sequence[EnvVarSecret]] = None,
|
||||
) -> Job[FunctionVersion]:
|
||||
job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets))
|
||||
return Job(job)
|
||||
|
||||
@override
|
||||
def get_function(self, name: str, *, version: str) -> FunctionVersion:
|
||||
@@ -761,6 +777,26 @@ class RemoteDBConnection(DBConnection):
|
||||
def drop_function(self, name: str, *, version: str) -> bool:
|
||||
return LOOP.run(self._conn.drop_function(name, version=version))
|
||||
|
||||
@override
|
||||
def create_secret(self, name: str, value: str) -> None:
|
||||
LOOP.run(self._conn.create_secret(name, value))
|
||||
|
||||
@override
|
||||
def alter_secret(self, name: str, value: str) -> None:
|
||||
LOOP.run(self._conn.alter_secret(name, value))
|
||||
|
||||
@override
|
||||
def describe_secret(self, name: str) -> SecretInfo:
|
||||
return LOOP.run(self._conn.describe_secret(name))
|
||||
|
||||
@override
|
||||
def list_secrets(self) -> List[str]:
|
||||
return LOOP.run(self._conn.list_secrets())
|
||||
|
||||
@override
|
||||
def drop_secret(self, name: str) -> None:
|
||||
LOOP.run(self._conn.drop_secret(name))
|
||||
|
||||
@override
|
||||
def list_jobs(self) -> List["JobInfo"]:
|
||||
"""List server-side jobs across the database's tables."""
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Named Secrets, and the bindings that deliver them to Functions.
|
||||
|
||||
A Secret is a database-scoped named credential. Nothing in this module holds a
|
||||
value: :class:`EnvVarSecret` names one and says which environment variable it
|
||||
should arrive in, and the value is resolved by the remote service when a
|
||||
Function bound to it runs. No API returns a stored credential, by construction
|
||||
rather than by policy -- there is no code path that could.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_SECRET_NAME = re.compile(r"^[A-Za-z0-9_-]{1,255}$")
|
||||
_ENV_VARIABLE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
def validate_secret_name(name: str) -> str:
|
||||
"""Check a Secret name locally and return it unchanged."""
|
||||
if not isinstance(name, str):
|
||||
raise TypeError(f"Secret name must be a string, not {type(name).__name__}")
|
||||
if not _SECRET_NAME.fullmatch(name):
|
||||
raise ValueError(f"invalid Secret name: {name!r}")
|
||||
return name
|
||||
|
||||
|
||||
def validate_env_variable(name: str) -> str:
|
||||
"""Check an environment variable name locally and return it unchanged."""
|
||||
if not isinstance(name, str):
|
||||
raise TypeError(
|
||||
f"environment variable name must be a string, not {type(name).__name__}"
|
||||
)
|
||||
if not _ENV_VARIABLE.fullmatch(name):
|
||||
raise ValueError(f"invalid environment variable name: {name!r}")
|
||||
return name
|
||||
|
||||
|
||||
class EnvVarSecret:
|
||||
"""A Secret bound to the environment variable a Function's library reads.
|
||||
|
||||
Pass these in the ``secrets`` sequence of
|
||||
[DBConnection.create_function][lancedb.db.DBConnection.create_function]. The
|
||||
Function's source is unchanged by binding: it reads ``OPENAI_API_KEY`` the
|
||||
way it always did, and the binding is what puts a value there.
|
||||
|
||||
This is a local value. Constructing it contacts no server, so it always
|
||||
succeeds and says nothing about whether the Secret exists; that is checked
|
||||
at registration, where a mistyped Secret name surfaces as a clear "does not
|
||||
exist" naming both the Secret and the variable bound to it. A mistyped
|
||||
*variable* name cannot be caught anywhere -- nothing knows which variables a
|
||||
Function reads -- so it surfaces on the first rows instead.
|
||||
|
||||
The type exists so a credential cannot be passed by accident. A bare string
|
||||
in the same position is a plausible-looking mistake with the opposite
|
||||
meaning, and it reads identically in a diff.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
secret : str
|
||||
The Secret's database-scoped name.
|
||||
env_variable : str
|
||||
The environment variable the Function reads it from.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from lancedb import EnvVarSecret
|
||||
>>> binding = EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
|
||||
>>> binding.secret, binding.env_variable
|
||||
('openai-prod', 'OPENAI_API_KEY')
|
||||
"""
|
||||
|
||||
__slots__ = ("_secret", "_env_variable")
|
||||
|
||||
def __init__(self, secret: str, env_variable: str):
|
||||
self._secret = validate_secret_name(secret)
|
||||
self._env_variable = validate_env_variable(env_variable)
|
||||
|
||||
@property
|
||||
def secret(self) -> str:
|
||||
"""The Secret's database-scoped name."""
|
||||
return self._secret
|
||||
|
||||
@property
|
||||
def env_variable(self) -> str:
|
||||
"""The environment variable the value is delivered in."""
|
||||
return self._env_variable
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"EnvVarSecret(secret={self._secret!r}, "
|
||||
f"env_variable={self._env_variable!r})"
|
||||
)
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return (
|
||||
isinstance(other, EnvVarSecret)
|
||||
and other._secret == self._secret
|
||||
and other._env_variable == self._env_variable
|
||||
)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((EnvVarSecret, self._secret, self._env_variable))
|
||||
|
||||
|
||||
class SecretInfo:
|
||||
"""What a database records about a Secret. Never its value.
|
||||
|
||||
Returned by
|
||||
[DBConnection.describe_secret][lancedb.db.DBConnection.describe_secret].
|
||||
"""
|
||||
|
||||
__slots__ = ("_name", "_created_at", "_updated_at")
|
||||
|
||||
def __init__(self, name: str, created_at: str, updated_at: str):
|
||||
self._name = name
|
||||
self._created_at = created_at
|
||||
self._updated_at = updated_at
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""The Secret's database-scoped name."""
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def created_at(self) -> str:
|
||||
"""When the Secret was created, as an RFC 3339 timestamp."""
|
||||
return self._created_at
|
||||
|
||||
@property
|
||||
def updated_at(self) -> str:
|
||||
"""When the Secret's value was last rotated, as an RFC 3339 timestamp."""
|
||||
return self._updated_at
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, value: dict) -> "SecretInfo":
|
||||
return cls(
|
||||
name=value["name"],
|
||||
created_at=value["created_at"],
|
||||
updated_at=value["updated_at"],
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"SecretInfo(name={self._name!r}, created_at={self._created_at!r}, "
|
||||
f"updated_at={self._updated_at!r})"
|
||||
)
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
return (
|
||||
isinstance(other, SecretInfo)
|
||||
and other._name == self._name
|
||||
and other._created_at == self._created_at
|
||||
and other._updated_at == self._updated_at
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EnvVarSecret",
|
||||
"SecretInfo",
|
||||
"validate_env_variable",
|
||||
"validate_secret_name",
|
||||
]
|
||||
@@ -37,6 +37,22 @@ def job_result(name: str) -> dict:
|
||||
return json.loads(fixture(name))["result"]
|
||||
|
||||
|
||||
def assert_no_secret_values(value):
|
||||
"""No client value models a resolved credential, at any nesting depth."""
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
assert key not in {
|
||||
"secret_value",
|
||||
"secret_values",
|
||||
"resolved_secret",
|
||||
"resolved_secrets",
|
||||
}
|
||||
assert_no_secret_values(child)
|
||||
elif isinstance(value, list):
|
||||
for child in value:
|
||||
assert_no_secret_values(child)
|
||||
|
||||
|
||||
def test_public_function_values_are_in_api_reference():
|
||||
docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md"
|
||||
rendered = docs.read_text()
|
||||
@@ -94,6 +110,7 @@ def test_function_version_identity_is_immutable_and_exact():
|
||||
version = FunctionVersion.from_json(json.dumps(value))
|
||||
assert version.name == "embed"
|
||||
assert version.version == "fv_01K3EXACT"
|
||||
assert dict(version.secret_bindings) == {"HF_TOKEN": "hf-prod"}
|
||||
|
||||
with pytest.raises((TypeError, ValueError)):
|
||||
version.version = "fv_changed"
|
||||
@@ -276,6 +293,25 @@ def test_refresh_result_rejects_non_u64_values(field):
|
||||
RefreshColumnResult.from_json(json.dumps(value))
|
||||
|
||||
|
||||
def test_canonical_client_values_carry_bindings_and_no_credentials():
|
||||
"""A binding names a Secret; the credential behind it has no client field."""
|
||||
version = FunctionVersion.from_json(
|
||||
json.dumps(job_result("remote_function_job.json"))
|
||||
)
|
||||
canonical = json.loads(version.to_canonical_json())
|
||||
assert canonical["secret_bindings"] == {"HF_TOKEN": "hf-prod"}
|
||||
assert_no_secret_values(canonical)
|
||||
|
||||
|
||||
def test_a_version_without_bindings_keeps_the_original_wire_shape():
|
||||
"""Every Function registered before Secrets existed serializes unchanged."""
|
||||
value = job_result("remote_function_job.json")
|
||||
del value["secret_bindings"]
|
||||
version = FunctionVersion.from_json(json.dumps(value))
|
||||
assert dict(version.secret_bindings) == {}
|
||||
assert "secret_bindings" not in json.loads(version.to_canonical_json())
|
||||
|
||||
|
||||
class _FunctionDeclarationInner:
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
@@ -28,6 +28,7 @@ from lancedb.functions import (
|
||||
_GRAMMAR_PRIMITIVES,
|
||||
udf,
|
||||
)
|
||||
from lancedb.secrets import EnvVarSecret
|
||||
|
||||
THRESHOLD = 20
|
||||
_CACHE = None
|
||||
@@ -53,6 +54,15 @@ def normalize_score(value: float) -> float:
|
||||
return value / 100.0
|
||||
|
||||
|
||||
@udf(
|
||||
pip=["openai==3.7.0"],
|
||||
env={"MODE": "test"},
|
||||
python_version="3.12",
|
||||
)
|
||||
def analyze_caption(caption: str) -> str:
|
||||
return caption.strip()
|
||||
|
||||
|
||||
def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
|
||||
assert isinstance(normalize_score, UdfDefinition)
|
||||
assert normalize_score(25.0) == 0.25
|
||||
@@ -69,6 +79,98 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable():
|
||||
}
|
||||
|
||||
|
||||
def test_secret_bound_udf_matches_its_shared_registration_golden():
|
||||
assert analyze_caption(" hello ") == "hello"
|
||||
bound = analyze_caption.bind_secrets(
|
||||
[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")]
|
||||
)
|
||||
assert (
|
||||
bound.to_canonical_json()
|
||||
== (FIXTURES / "remote_function_secret_registration_request.canonical.json")
|
||||
.read_text()
|
||||
.strip()
|
||||
)
|
||||
|
||||
|
||||
def test_an_unbound_request_carries_no_binding_at_all():
|
||||
"""Binding is a registration-time decision, so the definition holds none.
|
||||
|
||||
The decorator declares nothing about secrets, which is what makes the PRD's
|
||||
claim true: a Function's source and its registration request are identical
|
||||
whether or not a credential is later bound to it.
|
||||
"""
|
||||
unbound = json.loads(analyze_caption.registration_request.to_canonical_json())
|
||||
assert "secret_bindings" not in unbound
|
||||
assert "OPENAI_API_KEY" not in json.dumps(unbound)
|
||||
|
||||
|
||||
def test_binding_a_secret_leaves_the_packaged_artifact_untouched():
|
||||
"""The artifact is source bytes and nothing else, with or without secrets."""
|
||||
bound = analyze_caption.bind_secrets(
|
||||
[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")]
|
||||
)
|
||||
assert bound.artifact == analyze_caption.registration_request.artifact
|
||||
assert bound.artifact.digest == analyze_caption.registration_request.artifact.digest
|
||||
|
||||
|
||||
def test_a_function_declaring_no_secret_is_registered_exactly_as_before():
|
||||
"""The compatibility claim: nothing about the no-secret path moves."""
|
||||
assert (
|
||||
normalize_score.bind_secrets(None).to_canonical_json()
|
||||
== normalize_score.registration_request.to_canonical_json()
|
||||
)
|
||||
assert (
|
||||
"secret_bindings"
|
||||
not in normalize_score.registration_request.to_canonical_json()
|
||||
)
|
||||
|
||||
|
||||
def test_a_function_binds_each_variable_once():
|
||||
with pytest.raises(ValueError, match="binds each environment variable once"):
|
||||
analyze_caption.bind_secrets(
|
||||
[
|
||||
EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY"),
|
||||
EnvVarSecret(secret="openai-staging", env_variable="OPENAI_API_KEY"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_bindings_may_not_collide_with_plain_configuration():
|
||||
"""`env` is stored with the Function; a Secret is not. Refuse, do not pick."""
|
||||
with pytest.raises(ValueError, match="must be disjoint"):
|
||||
analyze_caption.bind_secrets(
|
||||
[EnvVarSecret(secret="mode-prod", env_variable="MODE")]
|
||||
)
|
||||
|
||||
|
||||
def test_a_function_binds_at_most_sixteen_secrets():
|
||||
bindings = [
|
||||
EnvVarSecret(secret=f"secret-{index}", env_variable=f"TOKEN_{index}")
|
||||
for index in range(17)
|
||||
]
|
||||
with pytest.raises(ValueError, match="at most 16 secrets"):
|
||||
normalize_score.bind_secrets(bindings)
|
||||
|
||||
|
||||
def test_a_credential_value_is_rejected_in_the_binding_position():
|
||||
"""The one mistake the typed binding exists to stop."""
|
||||
with pytest.raises(TypeError, match="EnvVarSecret"):
|
||||
analyze_caption.bind_secrets(["sk-live-0001"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("secret", "variable", "message"),
|
||||
[
|
||||
("openai-prod", "not-a-var", "invalid environment variable name"),
|
||||
("openai-prod", "API-TOKEN", "invalid environment variable name"),
|
||||
("not a name", "API_TOKEN", "invalid Secret name"),
|
||||
],
|
||||
)
|
||||
def test_a_binding_validates_both_names_locally(secret, variable, message):
|
||||
with pytest.raises(ValueError, match=message):
|
||||
EnvVarSecret(secret=secret, env_variable=variable)
|
||||
|
||||
|
||||
def _main_udf_source(
|
||||
*, threshold: int = 20, input_annotation: str = "int", comparison: str = ">="
|
||||
) -> str:
|
||||
@@ -1230,6 +1332,7 @@ def _mock_remote_function_catalog():
|
||||
"runtime": body["runtime"],
|
||||
"runtime_digest": "sha256:runtime",
|
||||
"environment_digest": "sha256:environment",
|
||||
"secret_bindings": body.get("secret_bindings", {}),
|
||||
"created_at": "2026-08-21T00:00:00Z",
|
||||
}
|
||||
response = {"job_id": "job-register"}
|
||||
@@ -1270,6 +1373,21 @@ def _mock_remote_function_catalog():
|
||||
"version": "fv_exact",
|
||||
}
|
||||
response = {"dropped": True}
|
||||
elif self.path in ("/v1/secrets/create", "/v1/secrets/alter"):
|
||||
assert set(body) == {"name", "value"}
|
||||
response = {}
|
||||
elif self.path == "/v1/secrets/list":
|
||||
if "page_token" not in body:
|
||||
response = {
|
||||
"secrets": [{"name": "openai-prod"}],
|
||||
"page_token": "next",
|
||||
}
|
||||
else:
|
||||
assert body["page_token"] == "next"
|
||||
response = {"secrets": [{"name": "hf-prod"}]}
|
||||
elif self.path == "/v1/secrets/drop":
|
||||
assert body == {"name": "openai-prod"}
|
||||
response = {}
|
||||
else:
|
||||
status = 404
|
||||
response = {"error": "not found"}
|
||||
@@ -1312,6 +1430,71 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
|
||||
)
|
||||
|
||||
|
||||
def test_remote_registration_sends_bindings_and_never_a_credential():
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = lancedb.connect(
|
||||
"db://dev",
|
||||
api_key="fake",
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
created = db.create_function(
|
||||
analyze_caption,
|
||||
secrets=[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")],
|
||||
)
|
||||
|
||||
assert dict(created.secret_bindings) == {"OPENAI_API_KEY": "openai-prod"}
|
||||
path, create_request = state["requests"][0]
|
||||
assert path == "/v1/functions/create"
|
||||
assert create_request["secret_bindings"] == {"OPENAI_API_KEY": "openai-prod"}
|
||||
# The request names a Secret and carries nothing that could be one.
|
||||
assert create_request == json.loads(
|
||||
analyze_caption.bind_secrets(
|
||||
[EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")]
|
||||
).to_canonical_json()
|
||||
)
|
||||
|
||||
|
||||
def test_remote_secret_verbs_round_trip():
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = lancedb.connect(
|
||||
"db://dev",
|
||||
api_key="fake",
|
||||
host_override=host,
|
||||
client_config={"retry_config": {"retries": 0}},
|
||||
)
|
||||
assert db.create_secret("openai-prod", "sk-live-0001") is None
|
||||
assert db.alter_secret("openai-prod", "sk-live-0002") is None
|
||||
assert db.list_secrets() == ["openai-prod", "hf-prod"]
|
||||
assert db.drop_secret("openai-prod") is None
|
||||
|
||||
routes = [path for path, _ in state["requests"]]
|
||||
assert routes == [
|
||||
"/v1/secrets/create",
|
||||
"/v1/secrets/alter",
|
||||
"/v1/secrets/list",
|
||||
"/v1/secrets/list",
|
||||
"/v1/secrets/drop",
|
||||
]
|
||||
assert state["requests"][0][1] == {"name": "openai-prod", "value": "sk-live-0001"}
|
||||
# The listing returns names, and the client has no way to ask for more.
|
||||
assert state["requests"][2][1] == {}
|
||||
|
||||
|
||||
def test_building_a_binding_contacts_no_server():
|
||||
"""A binding is a local value: it says nothing about whether the Secret exists.
|
||||
|
||||
Existence is the server's answer at registration, where a mistyped name is a
|
||||
clear error rather than a client-side check that was already stale.
|
||||
"""
|
||||
with _mock_remote_function_catalog() as (_host, state):
|
||||
binding = EnvVarSecret(secret="openai-prod", env_variable="OPENAI_API_KEY")
|
||||
assert binding.secret == "openai-prod"
|
||||
assert binding.env_variable == "OPENAI_API_KEY"
|
||||
|
||||
assert state["requests"] == []
|
||||
|
||||
|
||||
def test_blocking_remote_registration_returns_function_version():
|
||||
with _mock_remote_function_catalog() as (host, state):
|
||||
db = lancedb.connect(
|
||||
|
||||
Reference in New Issue
Block a user