diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 9219b67e6..1639a7c83 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -1023,3 +1023,41 @@ DROP STORAGE INTEGRATION customer_blobs; ``` Registration returns setup instructions and leaves `access_verified` false. It stores an account-qualified prefix such as `az://customer-data@customeraccount.blob.core.windows.net/raw/`. + +### GCP storage integrations + +Create one Secret containing the complete GCP application credentials JSON file +content, then reference its name when registering Google Cloud Storage access: + +```python +from pathlib import Path + +db.create_secret("gcp_credentials", Path("credentials.json").read_text()) +integration = db.create_storage_integration( + "customer_gcs", + "gs://customer-data/raw/", + storage_provider="GCP", + gcp_credentials_json_secret="gcp_credentials", +) +print(integration.configuration_instructions) +``` + +The equivalent SQL references the existing Secret: + +```sql +CREATE STORAGE INTEGRATION customer_gcs + STORAGE_PROVIDER = 'GCP' + STORAGE_PREFIX = 'gs://customer-data/raw/' + GCP_CREDENTIALS_JSON_SECRET = public.gcp_credentials; +DESCRIBE STORAGE INTEGRATION customer_gcs; +SHOW STORAGE INTEGRATIONS; +DROP STORAGE INTEGRATION customer_gcs; +``` + +`gcp_credentials_json_secret` names a Secret; pass the file content to +`create_secret`, not to `create_storage_integration`. Use `SecretReference` to +qualify another database or namespace. Sync and async connections accept the same +options. The service interprets the stored JSON when accessing storage; registration +does not validate its credentials format or verify access, and leaves +`access_verified` false. Integration metadata contains the Secret reference, never +its JSON content. GCP options cannot be combined with AWS or Azure options. diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 032fcb71e..1576348c9 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -394,11 +394,12 @@ class DBConnection(EnforceOverrides): aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None, aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None, aws_region: Optional[str] = None, - storage_provider: Literal["S3", "AZURE"] = "S3", + storage_provider: Literal["S3", "AZURE", "GCP"] = "S3", azure_storage_account: Optional[str] = None, azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, + gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: raise NotImplementedError @@ -2248,11 +2249,12 @@ class LanceDBConnection(DBConnection): aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None, aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None, aws_region: Optional[str] = None, - storage_provider: Literal["S3", "AZURE"] = "S3", + storage_provider: Literal["S3", "AZURE", "GCP"] = "S3", azure_storage_account: Optional[str] = None, azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, + gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: return LOOP.run( @@ -2268,6 +2270,7 @@ class LanceDBConnection(DBConnection): azure_tenant_id_secret=azure_tenant_id_secret, azure_client_id_secret=azure_client_id_secret, azure_client_secret_secret=azure_client_secret_secret, + gcp_credentials_json_secret=gcp_credentials_json_secret, namespace_path=namespace_path, ) ) @@ -3539,19 +3542,22 @@ class AsyncConnection(object): aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None, aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None, aws_region: Optional[str] = None, - storage_provider: Literal["S3", "AZURE"] = "S3", + storage_provider: Literal["S3", "AZURE", "GCP"] = "S3", azure_storage_account: Optional[str] = None, azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, + gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: - """Register S3 or Azure Blob access using database-owned credential objects. + """Register cloud storage access using database-owned credential Secrets. ``namespace_path`` follows Secret naming rules. A SecretReference can name another database; a string names a root Secret in this database. For Azure, set ``storage_provider="AZURE"``, ``azure_storage_account``, and Secret references for tenant ID, client ID and client secret. + For GCP, set ``storage_provider="GCP"`` and ``gcp_credentials_json_secret`` + to a Secret containing the application credentials JSON file content. Registration returns setup instructions without verifying storage access. """ azure_options = ( @@ -3559,7 +3565,23 @@ class AsyncConnection(object): azure_client_id_secret, azure_client_secret_secret, ) - if storage_provider == "AZURE": + if storage_provider != "GCP" and gcp_credentials_json_secret is not None: + raise ValueError("GCP credential options require storage_provider GCP") + if storage_provider == "GCP": + if gcp_credentials_json_secret is None: + raise ValueError("GCP requires gcp_credentials_json_secret") + if azure_storage_account is not None or any( + value is not None + for value in azure_options + + ( + aws_role_arn, + aws_access_key_id_secret, + aws_secret_access_key_secret, + aws_region, + ) + ): + raise ValueError("GCP integrations cannot contain AWS or Azure options") + elif storage_provider == "AZURE": if not azure_storage_account or any( value is None for value in azure_options ): @@ -3600,7 +3622,7 @@ class AsyncConnection(object): ) else: - raise ValueError("storage_provider must be S3 or AZURE") + raise ValueError("storage_provider must be S3, AZURE or GCP") def reference(value): if value is None: @@ -3622,6 +3644,7 @@ class AsyncConnection(object): "azure_tenant_id_secret": reference(azure_tenant_id_secret), "azure_client_id_secret": reference(azure_client_id_secret), "azure_client_secret_secret": reference(azure_client_secret_secret), + "gcp_credentials_json_secret": reference(gcp_credentials_json_secret), "aws_access_key_id_secret": reference(aws_access_key_id_secret), "aws_secret_access_key_secret": reference(aws_secret_access_key_secret), } diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 25aef11b9..c71f581ba 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -1048,11 +1048,12 @@ class RemoteDBConnection(DBConnection): aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None, aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None, aws_region: Optional[str] = None, - storage_provider: Literal["S3", "AZURE"] = "S3", + storage_provider: Literal["S3", "AZURE", "GCP"] = "S3", azure_storage_account: Optional[str] = None, azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_id_secret: Optional[Union[str, SecretReference]] = None, azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, + gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: return LOOP.run( @@ -1068,6 +1069,7 @@ class RemoteDBConnection(DBConnection): azure_tenant_id_secret=azure_tenant_id_secret, azure_client_id_secret=azure_client_id_secret, azure_client_secret_secret=azure_client_secret_secret, + gcp_credentials_json_secret=gcp_credentials_json_secret, namespace_path=namespace_path, ) ) diff --git a/python/python/lancedb/storage_integration.py b/python/python/lancedb/storage_integration.py index 025fbba12..4f443cdc6 100644 --- a/python/python/lancedb/storage_integration.py +++ b/python/python/lancedb/storage_integration.py @@ -30,7 +30,7 @@ class StorageIntegrationInfo: name: str database: str namespace_path: list[str] - storage_provider: Literal["S3", "AZURE"] + storage_provider: Literal["S3", "AZURE", "GCP"] storage_prefix: str aws_region: Optional[str] aws_account_id: str @@ -48,6 +48,7 @@ class StorageIntegrationInfo: azure_tenant_id_secret: Optional[SecretReference] = None azure_client_id_secret: Optional[SecretReference] = None azure_client_secret_secret: Optional[SecretReference] = None + gcp_credentials_json_secret: Optional[SecretReference] = None def __post_init__(self): for field in ( @@ -56,6 +57,7 @@ class StorageIntegrationInfo: "azure_tenant_id_secret", "azure_client_id_secret", "azure_client_secret_secret", + "gcp_credentials_json_secret", ): value = getattr(self, field) if isinstance(value, dict): diff --git a/python/python/tests/test_storage_integration.py b/python/python/tests/test_storage_integration.py index 345343848..d295aa98e 100644 --- a/python/python/tests/test_storage_integration.py +++ b/python/python/tests/test_storage_integration.py @@ -43,6 +43,25 @@ def remote_server(text_type=pa.string()): elif b"DROP STORAGE INTEGRATION" in command: result = pa.table({"dropped": [True]}) else: + if b"GCP_CREDENTIALS_JSON_SECRET" in command: + info.update( + storage_provider="GCP", + storage_prefix="gs://customer-data/raw/", + aws_role_arn=None, + aws_region=None, + aws_account_id="", + aws_iam_principal_arn="", + trust_policy=None, + permissions_policy=None, + configuration_instructions=( + "Grant the referenced identity access to the prefix." + ), + gcp_credentials_json_secret=dict( + database="shared", + namespace_path=["credentials"], + name="gcp", + ), + ) if b"STORAGE_AZURE_STORAGE_ACCOUNT" in command: info.update( storage_provider="AZURE", @@ -83,6 +102,7 @@ def remote_server(text_type=pa.string()): "azure_tenant_id_secret", "azure_client_id_secret", "azure_client_secret_secret", + "gcp_credentials_json_secret", ]: if field in row and row[field] is not None: row[field] = json.dumps(row[field]) @@ -305,3 +325,70 @@ async def test_async_azure_requires_account_and_all_three_secret_references(): ) assert info.storage_provider == "AZURE" assert b'STORAGE_AZURE_TENANT_ID_SECRET = "public"."tenant"' in calls[0] + + +@pytest.mark.parametrize( + "text_type", [pa.string(), pa.large_string(), pa.string_view()] +) +def test_gcp_lifecycle_uses_one_qualified_json_secret(text_type): + with remote_server(text_type) as (host, calls): + db = connect(host) + info = db.create_storage_integration( + "gcs", + "gs://customer-data/raw/", + storage_provider="GCP", + gcp_credentials_json_secret=SecretReference( + "gcp", database="shared", namespace_path=("credentials",) + ), + ) + assert info.storage_provider == "GCP" + assert info.gcp_credentials_json_secret.name == "gcp" + assert info.gcp_credentials_json_secret.database == "shared" + assert info.aws_role_arn is None + assert info.azure_storage_account is None + assert info.aws_iam_principal_arn == "" + assert info.trust_policy is None + assert not info.access_verified + assert db.describe_storage_integration("gcs") == info + assert list(db.list_storage_integrations()) == ["raw", "zeta"] + db.drop_storage_integration("gcs") + assert b"STORAGE_PROVIDER = 'GCP'" in calls[0] + assert b'GCP_CREDENTIALS_JSON_SECRET = "shared"."credentials"."gcp"' in calls[0] + assert b"STORAGE_AWS" not in calls[0] + assert b"STORAGE_AZURE" not in calls[0] + + +@pytest.mark.asyncio +async def test_async_gcp_requires_one_secret_and_rejects_other_cloud_options(): + with remote_server() as (host, calls): + db = await lancedb.connect_async( + "db://default", + api_key="fake", + sql_host_override=host, + host_override="http://127.0.0.1:1", + ) + for invalid in ( + {}, + {"gcp_credentials_json_secret": "gcp", "aws_role_arn": ROLE}, + {"gcp_credentials_json_secret": "gcp", "azure_storage_account": "account"}, + ): + with pytest.raises(ValueError): + await db.create_storage_integration( + "gcs", "gs://customer-data/raw/", storage_provider="GCP", **invalid + ) + with pytest.raises(ValueError): + await db.create_storage_integration( + "gcs", + "gs://customer-data/raw/", + aws_role_arn=ROLE, + gcp_credentials_json_secret="gcp", + ) + assert calls == [] + info = await db.create_storage_integration( + "gcs", + "gs://customer-data/raw/", + storage_provider="GCP", + gcp_credentials_json_secret="gcp", + ) + assert info.storage_provider == "GCP" + assert b'GCP_CREDENTIALS_JSON_SECRET = "public"."gcp"' in calls[0] diff --git a/rust/lancedb/src/remote/storage_integration.rs b/rust/lancedb/src/remote/storage_integration.rs index 51e0ccf8b..19ad5e6e9 100644 --- a/rust/lancedb/src/remote/storage_integration.rs +++ b/rust/lancedb/src/remote/storage_integration.rs @@ -54,6 +54,38 @@ pub(super) fn create_sql( namespace: &[String], request: &CreateStorageIntegrationRequest, ) -> Result { + if request.storage_provider != StorageProvider::Gcp + && request.gcp_credentials_json_secret.is_some() + { + return Err(invalid( + "GCP credential options require storage_provider GCP", + )); + } + if request.storage_provider == StorageProvider::Gcp { + if request.aws_role_arn.is_some() + || request.aws_access_key_id_secret.is_some() + || request.aws_secret_access_key_secret.is_some() + || request.aws_region.is_some() + || request.azure_storage_account.is_some() + || request.azure_tenant_id_secret.is_some() + || request.azure_client_id_secret.is_some() + || request.azure_client_secret_secret.is_some() + { + return Err(invalid( + "GCP integrations cannot contain AWS or Azure options", + )); + } + let reference = request + .gcp_credentials_json_secret + .as_ref() + .ok_or_else(|| invalid("GCP integrations require gcp_credentials_json_secret"))?; + return Ok(format!( + "CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}", + object_name(name, namespace)?, + literal(&request.storage_prefix), + secret_name(reference)? + )); + } if request.storage_provider == StorageProvider::Azure { if request.aws_role_arn.is_some() || request.aws_access_key_id_secret.is_some() @@ -172,6 +204,7 @@ pub(super) fn decode_info(batches: &[RecordBatch]) -> Result, #[serde(default, skip_serializing_if = "Option::is_none")] pub azure_client_secret_secret: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gcp_credentials_json_secret: Option, } /// Cloud storage providers supported by storage integrations. @@ -67,6 +69,8 @@ pub enum StorageProvider { S3, #[serde(rename = "AZURE")] Azure, + #[serde(rename = "GCP")] + Gcp, } impl CreateStorageIntegrationRequest { @@ -83,6 +87,7 @@ impl CreateStorageIntegrationRequest { azure_tenant_id_secret: None, azure_client_id_secret: None, azure_client_secret_secret: None, + gcp_credentials_json_secret: None, } } @@ -103,6 +108,7 @@ impl CreateStorageIntegrationRequest { azure_tenant_id_secret: None, azure_client_id_secret: None, azure_client_secret_secret: None, + gcp_credentials_json_secret: None, } } @@ -130,6 +136,7 @@ impl CreateStorageIntegrationRequest { azure_tenant_id_secret: Some(tenant_id), azure_client_id_secret: Some(client_id), azure_client_secret_secret: Some(client_secret), + gcp_credentials_json_secret: None, aws_role_arn: None, aws_access_key_id_secret: None, aws_secret_access_key_secret: None, @@ -137,6 +144,30 @@ impl CreateStorageIntegrationRequest { } } + /// Register Google Cloud Storage access using a Secret containing credentials JSON. + /// + /// ``` + /// use lancedb::storage_integration::{CreateStorageIntegrationRequest, SecretReference}; + /// let config = CreateStorageIntegrationRequest::gcp( + /// "gs://customer-data/raw/", SecretReference::new("gcp_credentials"), + /// ); + /// ``` + pub fn gcp(storage_prefix: impl Into, credentials_json: SecretReference) -> Self { + Self { + storage_provider: StorageProvider::Gcp, + storage_prefix: storage_prefix.into(), + gcp_credentials_json_secret: Some(credentials_json), + aws_role_arn: None, + aws_access_key_id_secret: None, + aws_secret_access_key_secret: None, + aws_region: None, + azure_storage_account: None, + azure_tenant_id_secret: None, + azure_client_id_secret: None, + azure_client_secret_secret: None, + } + } + /// Set the AWS region of the storage prefix. pub fn with_aws_region(mut self, region: impl Into) -> Self { self.aws_region = Some(region.into()); @@ -172,6 +203,8 @@ pub struct StorageIntegrationInfo { pub azure_client_id_secret: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub azure_client_secret_secret: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gcp_credentials_json_secret: Option, pub aws_account_id: String, pub aws_iam_principal_arn: String, pub aws_external_id: String,