rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: accept GCP credentials Secret in storage integration clients (#4771)

Browse Source 
Adds GCP inputs to the existing database-owned storage integration
methods. GCP takes one Secret reference; the Secret stores the complete
application credentials JSON as an opaque string. The same methods and
SQL transport handle create, describe, list and drop.

```python
info = db.create_storage_integration(
    "gcs", "gs://customer-data/raw/", storage_provider="GCP",
    gcp_credentials_json_secret=SecretReference(
        "gcp_json", database="shared", namespace_path=("credentials",),
    ),
)
print(info.configuration_instructions)
info = db.describe_storage_integration("gcs")
names = list(db.list_storage_integrations())
db.drop_storage_integration("gcs")
```

```sql
CREATE STORAGE INTEGRATION gcs
    STORAGE_PROVIDER = 'GCP'
    STORAGE_PREFIX = 'gs://customer-data/raw/'
    GCP_CREDENTIALS_JSON_SECRET = shared.credentials.gcp_json;
DESCRIBE STORAGE INTEGRATION gcs;
SHOW STORAGE INTEGRATIONS;
DROP STORAGE INTEGRATION gcs;
```

```mermaid
flowchart LR
    Python[Sync or async Python] --> SQL[Existing Flight SQL transport]
    SQL --> Config[Configuration and one Secret reference]
```

Adds Rust request construction, SQL rendering and typed metadata
decoding, plus sync/async Flight SQL regressions across the three Arrow
string encodings. Rejects missing or mixed-cloud credential options.
Local builds/tests are excluded by request; validation runs in CI.
This commit is contained in:
Jack Ye
2026-10-31 01:11:25 -07:00
committed by GitHub
parent b15fa9d6e4
commit 1fb2dd1551
7 changed files with 246 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/python/python.md
+38
View File
@@ -1023,3 +1023,41 @@ DROP STORAGE INTEGRATION customer_blobs;
```
Registration returns setup instructions and leaves `access_verified` false. It stores an account-qualified prefix such as `az://customer-data@customeraccount.blob.core.windows.net/raw/`.
### GCP storage integrations
Create one Secret containing the complete GCP application credentials JSON file
content, then reference its name when registering Google Cloud Storage access:
```python
from pathlib import Path
db.create_secret("gcp_credentials", Path("credentials.json").read_text())
integration = db.create_storage_integration(
"customer_gcs",
"gs://customer-data/raw/",
storage_provider="GCP",
gcp_credentials_json_secret="gcp_credentials",
)
print(integration.configuration_instructions)
```
The equivalent SQL references the existing Secret:
```sql
CREATE STORAGE INTEGRATION customer_gcs
STORAGE_PROVIDER = 'GCP'
STORAGE_PREFIX = 'gs://customer-data/raw/'
GCP_CREDENTIALS_JSON_SECRET = public.gcp_credentials;
DESCRIBE STORAGE INTEGRATION customer_gcs;
SHOW STORAGE INTEGRATIONS;
DROP STORAGE INTEGRATION customer_gcs;
```
`gcp_credentials_json_secret` names a Secret; pass the file content to
`create_secret`, not to `create_storage_integration`. Use `SecretReference` to
qualify another database or namespace. Sync and async connections accept the same
options. The service interprets the stored JSON when accessing storage; registration
does not validate its credentials format or verify access, and leaves
`access_verified` false. Integration metadata contains the Secret reference, never
its JSON content. GCP options cannot be combined with AWS or Azure options.
python/python/lancedb/db.py
+29 -6
View File
@@ -394,11 +394,12 @@ class DBConnection(EnforceOverrides):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
raise NotImplementedError
@@ -2248,11 +2249,12 @@ class LanceDBConnection(DBConnection):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -2268,6 +2270,7 @@ class LanceDBConnection(DBConnection):
azure_tenant_id_secret=azure_tenant_id_secret,
azure_client_id_secret=azure_client_id_secret,
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
namespace_path=namespace_path,
)
)
@@ -3539,19 +3542,22 @@ class AsyncConnection(object):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
"""Register S3 or Azure Blob access using database-owned credential objects.
"""Register cloud storage access using database-owned credential Secrets.
``namespace_path`` follows Secret naming rules. A SecretReference can name
another database; a string names a root Secret in this database.
For Azure, set ``storage_provider="AZURE"``, ``azure_storage_account``,
and Secret references for tenant ID, client ID and client secret.
For GCP, set ``storage_provider="GCP"`` and ``gcp_credentials_json_secret``
to a Secret containing the application credentials JSON file content.
Registration returns setup instructions without verifying storage access.
"""
azure_options = (
@@ -3559,7 +3565,23 @@ class AsyncConnection(object):
azure_client_id_secret,
azure_client_secret_secret,
)
if storage_provider == "AZURE":
if storage_provider != "GCP" and gcp_credentials_json_secret is not None:
raise ValueError("GCP credential options require storage_provider GCP")
if storage_provider == "GCP":
if gcp_credentials_json_secret is None:
raise ValueError("GCP requires gcp_credentials_json_secret")
if azure_storage_account is not None or any(
value is not None
for value in azure_options
+ (
aws_role_arn,
aws_access_key_id_secret,
aws_secret_access_key_secret,
aws_region,
)
):
raise ValueError("GCP integrations cannot contain AWS or Azure options")
elif storage_provider == "AZURE":
if not azure_storage_account or any(
value is None for value in azure_options
):
@@ -3600,7 +3622,7 @@ class AsyncConnection(object):
)
else:
raise ValueError("storage_provider must be S3 or AZURE")
raise ValueError("storage_provider must be S3, AZURE or GCP")
def reference(value):
if value is None:
@@ -3622,6 +3644,7 @@ class AsyncConnection(object):
"azure_tenant_id_secret": reference(azure_tenant_id_secret),
"azure_client_id_secret": reference(azure_client_id_secret),
"azure_client_secret_secret": reference(azure_client_secret_secret),
"gcp_credentials_json_secret": reference(gcp_credentials_json_secret),
"aws_access_key_id_secret": reference(aws_access_key_id_secret),
"aws_secret_access_key_secret": reference(aws_secret_access_key_secret),
}
python/python/lancedb/remote/db.py
+3 -1
View File
@@ -1048,11 +1048,12 @@ class RemoteDBConnection(DBConnection):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -1068,6 +1069,7 @@ class RemoteDBConnection(DBConnection):
azure_tenant_id_secret=azure_tenant_id_secret,
azure_client_id_secret=azure_client_id_secret,
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
namespace_path=namespace_path,
)
)
python/python/lancedb/storage_integration.py
+3 -1
View File
@@ -30,7 +30,7 @@ class StorageIntegrationInfo:
name: str
database: str
namespace_path: list[str]
storage_provider: Literal["S3", "AZURE"]
storage_provider: Literal["S3", "AZURE", "GCP"]
storage_prefix: str
aws_region: Optional[str]
aws_account_id: str
@@ -48,6 +48,7 @@ class StorageIntegrationInfo:
azure_tenant_id_secret: Optional[SecretReference] = None
azure_client_id_secret: Optional[SecretReference] = None
azure_client_secret_secret: Optional[SecretReference] = None
gcp_credentials_json_secret: Optional[SecretReference] = None
def __post_init__(self):
for field in (
@@ -56,6 +57,7 @@ class StorageIntegrationInfo:
"azure_tenant_id_secret",
"azure_client_id_secret",
"azure_client_secret_secret",
"gcp_credentials_json_secret",
):
value = getattr(self, field)
if isinstance(value, dict):
python/python/tests/test_storage_integration.py
+87
View File
@@ -43,6 +43,25 @@ def remote_server(text_type=pa.string()):
elif b"DROP STORAGE INTEGRATION" in command:
result = pa.table({"dropped": [True]})
else:
if b"GCP_CREDENTIALS_JSON_SECRET" in command:
info.update(
storage_provider="GCP",
storage_prefix="gs://customer-data/raw/",
aws_role_arn=None,
aws_region=None,
aws_account_id="",
aws_iam_principal_arn="",
trust_policy=None,
permissions_policy=None,
configuration_instructions=(
"Grant the referenced identity access to the prefix."
),
gcp_credentials_json_secret=dict(
database="shared",
namespace_path=["credentials"],
name="gcp",
),
)
if b"STORAGE_AZURE_STORAGE_ACCOUNT" in command:
info.update(
storage_provider="AZURE",
@@ -83,6 +102,7 @@ def remote_server(text_type=pa.string()):
"azure_tenant_id_secret",
"azure_client_id_secret",
"azure_client_secret_secret",
"gcp_credentials_json_secret",
]:
if field in row and row[field] is not None:
row[field] = json.dumps(row[field])
@@ -305,3 +325,70 @@ async def test_async_azure_requires_account_and_all_three_secret_references():
)
assert info.storage_provider == "AZURE"
assert b'STORAGE_AZURE_TENANT_ID_SECRET = "public"."tenant"' in calls[0]
@pytest.mark.parametrize(
"text_type", [pa.string(), pa.large_string(), pa.string_view()]
)
def test_gcp_lifecycle_uses_one_qualified_json_secret(text_type):
with remote_server(text_type) as (host, calls):
db = connect(host)
info = db.create_storage_integration(
"gcs",
"gs://customer-data/raw/",
storage_provider="GCP",
gcp_credentials_json_secret=SecretReference(
"gcp", database="shared", namespace_path=("credentials",)
),
)
assert info.storage_provider == "GCP"
assert info.gcp_credentials_json_secret.name == "gcp"
assert info.gcp_credentials_json_secret.database == "shared"
assert info.aws_role_arn is None
assert info.azure_storage_account is None
assert info.aws_iam_principal_arn == ""
assert info.trust_policy is None
assert not info.access_verified
assert db.describe_storage_integration("gcs") == info
assert list(db.list_storage_integrations()) == ["raw", "zeta"]
db.drop_storage_integration("gcs")
assert b"STORAGE_PROVIDER = 'GCP'" in calls[0]
assert b'GCP_CREDENTIALS_JSON_SECRET = "shared"."credentials"."gcp"' in calls[0]
assert b"STORAGE_AWS" not in calls[0]
assert b"STORAGE_AZURE" not in calls[0]
@pytest.mark.asyncio
async def test_async_gcp_requires_one_secret_and_rejects_other_cloud_options():
with remote_server() as (host, calls):
db = await lancedb.connect_async(
"db://default",
api_key="fake",
sql_host_override=host,
host_override="http://127.0.0.1:1",
)
for invalid in (
{},
{"gcp_credentials_json_secret": "gcp", "aws_role_arn": ROLE},
{"gcp_credentials_json_secret": "gcp", "azure_storage_account": "account"},
):
with pytest.raises(ValueError):
await db.create_storage_integration(
"gcs", "gs://customer-data/raw/", storage_provider="GCP", **invalid
)
with pytest.raises(ValueError):
await db.create_storage_integration(
"gcs",
"gs://customer-data/raw/",
aws_role_arn=ROLE,
gcp_credentials_json_secret="gcp",
)
assert calls == []
info = await db.create_storage_integration(
"gcs",
"gs://customer-data/raw/",
storage_provider="GCP",
gcp_credentials_json_secret="gcp",
)
assert info.storage_provider == "GCP"
assert b'GCP_CREDENTIALS_JSON_SECRET = "public"."gcp"' in calls[0]
rust/lancedb/src/remote/storage_integration.rs
+52
View File
@@ -54,6 +54,38 @@ pub(super) fn create_sql(
namespace: &[String],
request: &CreateStorageIntegrationRequest,
) -> Result<String> {
if request.storage_provider != StorageProvider::Gcp
&& request.gcp_credentials_json_secret.is_some()
{
return Err(invalid(
"GCP credential options require storage_provider GCP",
));
}
if request.storage_provider == StorageProvider::Gcp {
if request.aws_role_arn.is_some()
|| request.aws_access_key_id_secret.is_some()
|| request.aws_secret_access_key_secret.is_some()
|| request.aws_region.is_some()
|| request.azure_storage_account.is_some()
|| request.azure_tenant_id_secret.is_some()
|| request.azure_client_id_secret.is_some()
|| request.azure_client_secret_secret.is_some()
{
return Err(invalid(
"GCP integrations cannot contain AWS or Azure options",
));
}
let reference = request
.gcp_credentials_json_secret
.as_ref()
.ok_or_else(|| invalid("GCP integrations require gcp_credentials_json_secret"))?;
return Ok(format!(
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}",
object_name(name, namespace)?,
literal(&request.storage_prefix),
secret_name(reference)?
));
}
if request.storage_provider == StorageProvider::Azure {
if request.aws_role_arn.is_some()
|| request.aws_access_key_id_secret.is_some()
@@ -172,6 +204,7 @@ pub(super) fn decode_info(batches: &[RecordBatch]) -> Result<StorageIntegrationI
| "azure_tenant_id_secret"
| "azure_client_id_secret"
| "azure_client_secret_secret"
| "gcp_credentials_json_secret"
| "trust_policy"
| "permissions_policy"
) && let Some(json) = value.as_str()
@@ -233,6 +266,25 @@ mod tests {
assert!(create_sql("a/b", &[], &role).is_err());
}
#[test]
fn gcp_sql_accepts_one_secret_object_and_rejects_mixed_modes() {
let mut request = CreateStorageIntegrationRequest::gcp(
"gs://customer-data/raw/",
SecretReference::new("gcp")
.with_database("shared")
.with_namespace(vec!["credentials".into()]),
);
let sql = create_sql("gcs", &[], &request).unwrap();
assert!(sql.contains("STORAGE_PROVIDER = 'GCP'"));
assert!(sql.contains("GCP_CREDENTIALS_JSON_SECRET = \"shared\".\"credentials\".\"gcp\""));
assert!(!sql.contains("STORAGE_AWS"));
request.aws_region = Some("us-east-1".into());
assert!(create_sql("gcs", &[], &request).is_err());
request.aws_region = None;
request.gcp_credentials_json_secret = None;
assert!(create_sql("gcs", &[], &request).is_err());
}
#[test]
fn azure_sql_uses_only_explicit_account_and_three_secret_objects() {
let reference = SecretReference::new("credential")
rust/lancedb/src/storage_integration.rs
+34 -1
View File
@@ -36,7 +36,7 @@ impl SecretReference {
}
}
/// Configuration for registering an AWS S3 or Azure Blob storage integration.
/// Configuration for registering a cloud storage integration.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct CreateStorageIntegrationRequest {
@@ -58,6 +58,8 @@ pub struct CreateStorageIntegrationRequest {
pub azure_client_id_secret: Option<SecretReference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub azure_client_secret_secret: Option<SecretReference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gcp_credentials_json_secret: Option<SecretReference>,
}
/// Cloud storage providers supported by storage integrations.
@@ -67,6 +69,8 @@ pub enum StorageProvider {
S3,
#[serde(rename = "AZURE")]
Azure,
#[serde(rename = "GCP")]
Gcp,
}
impl CreateStorageIntegrationRequest {
@@ -83,6 +87,7 @@ impl CreateStorageIntegrationRequest {
azure_tenant_id_secret: None,
azure_client_id_secret: None,
azure_client_secret_secret: None,
gcp_credentials_json_secret: None,
}
}
@@ -103,6 +108,7 @@ impl CreateStorageIntegrationRequest {
azure_tenant_id_secret: None,
azure_client_id_secret: None,
azure_client_secret_secret: None,
gcp_credentials_json_secret: None,
}
}
@@ -130,6 +136,7 @@ impl CreateStorageIntegrationRequest {
azure_tenant_id_secret: Some(tenant_id),
azure_client_id_secret: Some(client_id),
azure_client_secret_secret: Some(client_secret),
gcp_credentials_json_secret: None,
aws_role_arn: None,
aws_access_key_id_secret: None,
aws_secret_access_key_secret: None,
@@ -137,6 +144,30 @@ impl CreateStorageIntegrationRequest {
}
}
/// Register Google Cloud Storage access using a Secret containing credentials JSON.
///
/// ```
/// use lancedb::storage_integration::{CreateStorageIntegrationRequest, SecretReference};
/// let config = CreateStorageIntegrationRequest::gcp(
/// "gs://customer-data/raw/", SecretReference::new("gcp_credentials"),
/// );
/// ```
pub fn gcp(storage_prefix: impl Into<String>, credentials_json: SecretReference) -> Self {
Self {
storage_provider: StorageProvider::Gcp,
storage_prefix: storage_prefix.into(),
gcp_credentials_json_secret: Some(credentials_json),
aws_role_arn: None,
aws_access_key_id_secret: None,
aws_secret_access_key_secret: None,
aws_region: None,
azure_storage_account: None,
azure_tenant_id_secret: None,
azure_client_id_secret: None,
azure_client_secret_secret: None,
}
}
/// Set the AWS region of the storage prefix.
pub fn with_aws_region(mut self, region: impl Into<String>) -> Self {
self.aws_region = Some(region.into());
@@ -172,6 +203,8 @@ pub struct StorageIntegrationInfo {
pub azure_client_id_secret: Option<SecretReference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub azure_client_secret_secret: Option<SecretReference>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gcp_credentials_json_secret: Option<SecretReference>,
pub aws_account_id: String,
pub aws_iam_principal_arn: String,
pub aws_external_id: String,