Files
lancedb/python/src/authz.rs
T
Jack Ye 6d126a3601 feat: add database storage integration APIs (#4618)
Adds database-owned S3 storage integrations with Secret-style namespace
paths in a separate keyspace. Use a regular database under the cluster
root for bootstrap integrations and Secrets. `default` is an example
name, with no enforced or automatically provisioned database; `system`
remains read-only. Authentication uses either an IAM role or two
existing Secret references, including cross-database references;
configuration metadata never contains credential values.

### Python experience

```python
catalog = lancedb.connect_catalog("https://my-server.example", api_key="secret")
db = catalog.connect_database("default")
info = db.create_storage_integration(
    "raw", "s3://customer-bucket/raw/",
    aws_role_arn="arn:aws:iam::123456789012:role/lancedb-access",
    aws_region="us-east-1",
)
print(info.configuration_instructions)
info = db.describe_storage_integration("raw")
print(list(db.list_storage_integrations()))
db.drop_storage_integration("raw")
```

For static credentials, replace `aws_role_arn` with
`aws_access_key_id_secret=SecretReference("s3_key_id",
database="default")` and
`aws_secret_access_key_secret=SecretReference("s3_secret_key",
database="default")`, imported from `lancedb.storage_integration`. Pass
`namespace_path` to address nested namespaces. Async database
connections expose the same APIs; listing is lazy. All four client
operations execute SQL through the existing SQL transport, with typed
Arrow result decoding and name-based listing cursors.

### SQL experience

```sql
CREATE STORAGE INTEGRATION default.public.raw
    STORAGE_PROVIDER = 'S3'
    STORAGE_PREFIX = 's3://customer-bucket/raw/'
    STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::123456789012:role/lancedb-access'
    STORAGE_AWS_REGION = 'us-east-1';
DESCRIBE STORAGE INTEGRATION default.public.raw;
SHOW STORAGE INTEGRATIONS IN default.public;
DROP STORAGE INTEGRATION IF EXISTS default.public.raw;

CREATE STORAGE INTEGRATION default.public.raw
    STORAGE_PROVIDER = 'S3'
    STORAGE_PREFIX = 's3://customer-bucket/raw/'
    STORAGE_AWS_ACCESS_KEY_ID_SECRET = default.public.s3_key_id
    STORAGE_AWS_SECRET_ACCESS_KEY_SECRET = default.public.s3_secret_key;
```

CREATE and DESCRIBE return configuration and a single sentence of setup
guidance. IAM mode also returns the deployment role principal, stable
external ID, customer trust policy, and prefix-scoped permissions
policy. Registration does not verify access. Creation requires namespace
CREATE_STORAGE_INTEGRATION plus parent USAGE; Secret references require
source database, namespace, and Secret USAGE. Describe requires object
USAGE; drop requires OWNERSHIP. SQL supports GRANT/REVOKE for these
objects and their creation privilege.

Includes client documentation and Flight SQL regression coverage; no
dedicated storage-integration REST endpoints are used. Local
builds/tests are excluded by request.
2026-10-30 00:26:42 -07:00

294 lines
9.7 KiB
Rust

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use crate::error::PythonErrorExt;
use crate::runtime::future_into_py;
use lancedb::authz::*;
use pyo3::exceptions::{PyRuntimeError, PyValueError};
use pyo3::types::{PyAnyMethods, PyBytes, PyString};
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
use std::sync::Arc;
#[pyclass]
pub struct AuthorizationClient {
pub(crate) inner: Arc<dyn Authorization>,
}
#[pymethods]
impl AuthorizationClient {
fn list_acls<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: ListAccessControlEntriesRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.list_acls(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
fn add_acl<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: AccessControlEntryRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.add_acl(request).await.infer_error()
})
}
fn delete_acl<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: AccessControlEntryRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.delete_acl(request).await.infer_error()
})
}
fn list_role_bindings<'py>(
self_: PyRef<'py, Self>,
request: &str,
) -> PyResult<Bound<'py, PyAny>> {
let request: ListRoleBindingsRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.list_role_bindings(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
fn add_role_binding<'py>(
self_: PyRef<'py, Self>,
request: &str,
) -> PyResult<Bound<'py, PyAny>> {
let request: RoleBindingRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.add_role_binding(request).await.infer_error()
})
}
fn delete_role_binding<'py>(
self_: PyRef<'py, Self>,
request: &str,
) -> PyResult<Bound<'py, PyAny>> {
let request: RoleBindingRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.delete_role_binding(request).await.infer_error()
})
}
fn get_principal<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: GetPrincipalRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.get_principal(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
fn get_group<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: GetGroupRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.get_group(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
}
/// The shared privilege parser and canonical wire names.
#[pyclass(frozen)]
pub struct AuthzPrivilege {
inner: Privilege,
}
#[pymethods]
impl AuthzPrivilege {
#[new]
fn new(value: &str) -> PyResult<Self> {
Ok(Self {
inner: value.parse().map_err(PyValueError::new_err)?,
})
}
#[staticmethod]
fn known() -> Vec<String> {
Privilege::KNOWN.iter().map(ToString::to_string).collect()
}
fn __str__(&self) -> String {
self.inner.to_string()
}
}
#[pyclass(frozen)]
pub struct AuthzSubject {
inner: Subject,
}
#[pymethods]
impl AuthzSubject {
#[new]
fn new(kind: &str, value: &Bound<'_, PyAny>) -> PyResult<Self> {
let value: String = value
.extract()
.map_err(|_| PyValueError::new_err("Subject value must be a nonempty string"))?;
Ok(Self {
inner: Subject::new(kind.parse().infer_error()?, value).infer_error()?,
})
}
#[staticmethod]
fn from_canonical(value: &str) -> PyResult<Self> {
Ok(Self {
inner: Subject::from_canonical(value).infer_error()?,
})
}
#[getter]
fn kind(&self) -> &str {
self.inner.kind().as_str()
}
#[getter]
fn value(&self) -> &str {
self.inner.value()
}
#[getter]
fn display_value(&self) -> &str {
self.inner.display_value()
}
#[pyo3(signature = (context=None))]
fn to_wire(&self, context: Option<&str>) -> PyResult<String> {
match context {
None => {}
Some("principal") => self.inner.validate_principal().infer_error()?,
Some("group") => self.inner.validate_group().infer_error()?,
Some("role_member") => self.inner.validate_role_member().infer_error()?,
Some(_) => return Err(PyValueError::new_err("Invalid subject context")),
}
Ok(self.inner.to_wire())
}
fn __repr__(&self) -> String {
format!("{:?}", self.inner)
}
fn __str__(&self) -> String {
self.inner.to_string()
}
}
#[pyclass(frozen)]
pub struct AuthzObject {
inner: Object,
}
impl AuthzObject {
fn validated(inner: Object) -> PyResult<Self> {
inner.validate().map_err(PyValueError::new_err)?;
Ok(Self { inner })
}
}
fn namespace_path(value: &Bound<'_, PyAny>) -> PyResult<NamespacePath> {
if value.is_instance_of::<PyString>() || value.is_instance_of::<PyBytes>() {
return Err(PyValueError::new_err(
"namespace must be a sequence of components",
));
}
let components = value
.extract()
.map_err(|_| PyValueError::new_err("namespace must be a sequence of string components"))?;
NamespacePath::new(components).map_err(PyValueError::new_err)
}
#[pymethods]
impl AuthzObject {
#[new]
fn new(value: &str) -> PyResult<Self> {
Ok(Self {
inner: value.parse().infer_error()?,
})
}
#[staticmethod]
fn system() -> Self {
Self {
inner: Object::System,
}
}
#[staticmethod]
fn database(name: &str) -> PyResult<Self> {
Self::validated(Object::Database(DatabaseObject::new(name.into())))
}
#[staticmethod]
fn namespace(database: &str, namespace: &Bound<'_, PyAny>) -> PyResult<Self> {
Self::validated(Object::Namespace(NamespaceObject {
database: database.into(),
namespace: namespace_path(namespace)?,
}))
}
#[staticmethod]
fn table(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::Table(TableObject {
database: database.into(),
namespace: namespace_path(namespace)?,
table: name.into(),
}))
}
#[staticmethod]
fn view(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::View(ViewObject {
database: database.into(),
namespace: namespace_path(namespace)?,
view: name.into(),
}))
}
#[staticmethod]
fn secret(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::Secret(SecretObject {
database: database.into(),
namespace: namespace_path(namespace)?,
secret: name.into(),
}))
}
#[staticmethod]
fn storage_integration(
database: &str,
namespace: &Bound<'_, PyAny>,
name: &str,
) -> PyResult<Self> {
Self::validated(Object::StorageIntegration(StorageIntegrationObject {
database: database.into(),
namespace: namespace_path(namespace)?,
storage_integration: name.into(),
}))
}
#[staticmethod]
fn function(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::Function(FunctionObject {
database: database.into(),
namespace: namespace_path(namespace)?,
function: name.into(),
}))
}
fn __str__(&self) -> String {
self.inner.to_string()
}
fn __repr__(&self) -> String {
format!("{:?}", self.inner)
}
}