rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: accept GCP credentials Secret in storage integration clients (#4771)

Browse Source 
Adds GCP inputs to the existing database-owned storage integration
methods. GCP takes one Secret reference; the Secret stores the complete
application credentials JSON as an opaque string. The same methods and
SQL transport handle create, describe, list and drop.

```python
info = db.create_storage_integration(
    "gcs", "gs://customer-data/raw/", storage_provider="GCP",
    gcp_credentials_json_secret=SecretReference(
        "gcp_json", database="shared", namespace_path=("credentials",),
    ),
)
print(info.configuration_instructions)
info = db.describe_storage_integration("gcs")
names = list(db.list_storage_integrations())
db.drop_storage_integration("gcs")
```

```sql
CREATE STORAGE INTEGRATION gcs
    STORAGE_PROVIDER = 'GCP'
    STORAGE_PREFIX = 'gs://customer-data/raw/'
    GCP_CREDENTIALS_JSON_SECRET = shared.credentials.gcp_json;
DESCRIBE STORAGE INTEGRATION gcs;
SHOW STORAGE INTEGRATIONS;
DROP STORAGE INTEGRATION gcs;
```

```mermaid
flowchart LR
    Python[Sync or async Python] --> SQL[Existing Flight SQL transport]
    SQL --> Config[Configuration and one Secret reference]
```

Adds Rust request construction, SQL rendering and typed metadata
decoding, plus sync/async Flight SQL regressions across the three Arrow
string encodings. Rejects missing or mixed-cloud credential options.
Local builds/tests are excluded by request; validation runs in CI.
This commit is contained in:
Jack Ye
2026-10-31 01:11:25 -07:00
committed by GitHub
parent b15fa9d6e4
commit 1fb2dd1551
7 changed files with 246 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/db.py
+29 -6
View File
@@ -394,11 +394,12 @@ class DBConnection(EnforceOverrides):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
raise NotImplementedError
@@ -2248,11 +2249,12 @@ class LanceDBConnection(DBConnection):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -2268,6 +2270,7 @@ class LanceDBConnection(DBConnection):
azure_tenant_id_secret=azure_tenant_id_secret,
azure_client_id_secret=azure_client_id_secret,
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
namespace_path=namespace_path,
)
)
@@ -3539,19 +3542,22 @@ class AsyncConnection(object):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
"""Register S3 or Azure Blob access using database-owned credential objects.
"""Register cloud storage access using database-owned credential Secrets.
``namespace_path`` follows Secret naming rules. A SecretReference can name
another database; a string names a root Secret in this database.
For Azure, set ``storage_provider="AZURE"``, ``azure_storage_account``,
and Secret references for tenant ID, client ID and client secret.
For GCP, set ``storage_provider="GCP"`` and ``gcp_credentials_json_secret``
to a Secret containing the application credentials JSON file content.
Registration returns setup instructions without verifying storage access.
"""
azure_options = (
@@ -3559,7 +3565,23 @@ class AsyncConnection(object):
azure_client_id_secret,
azure_client_secret_secret,
)
if storage_provider == "AZURE":
if storage_provider != "GCP" and gcp_credentials_json_secret is not None:
raise ValueError("GCP credential options require storage_provider GCP")
if storage_provider == "GCP":
if gcp_credentials_json_secret is None:
raise ValueError("GCP requires gcp_credentials_json_secret")
if azure_storage_account is not None or any(
value is not None
for value in azure_options
+ (
aws_role_arn,
aws_access_key_id_secret,
aws_secret_access_key_secret,
aws_region,
)
):
raise ValueError("GCP integrations cannot contain AWS or Azure options")
elif storage_provider == "AZURE":
if not azure_storage_account or any(
value is None for value in azure_options
):
@@ -3600,7 +3622,7 @@ class AsyncConnection(object):
)
else:
raise ValueError("storage_provider must be S3 or AZURE")
raise ValueError("storage_provider must be S3, AZURE or GCP")
def reference(value):
if value is None:
@@ -3622,6 +3644,7 @@ class AsyncConnection(object):
"azure_tenant_id_secret": reference(azure_tenant_id_secret),
"azure_client_id_secret": reference(azure_client_id_secret),
"azure_client_secret_secret": reference(azure_client_secret_secret),
"gcp_credentials_json_secret": reference(gcp_credentials_json_secret),
"aws_access_key_id_secret": reference(aws_access_key_id_secret),
"aws_secret_access_key_secret": reference(aws_secret_access_key_secret),
}
python/python/lancedb/remote/db.py
+3 -1
View File
@@ -1048,11 +1048,12 @@ class RemoteDBConnection(DBConnection):
aws_access_key_id_secret: Optional[Union[str, SecretReference]] = None,
aws_secret_access_key_secret: Optional[Union[str, SecretReference]] = None,
aws_region: Optional[str] = None,
storage_provider: Literal["S3", "AZURE"] = "S3",
storage_provider: Literal["S3", "AZURE", "GCP"] = "S3",
azure_storage_account: Optional[str] = None,
azure_tenant_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_id_secret: Optional[Union[str, SecretReference]] = None,
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -1068,6 +1069,7 @@ class RemoteDBConnection(DBConnection):
azure_tenant_id_secret=azure_tenant_id_secret,
azure_client_id_secret=azure_client_id_secret,
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
namespace_path=namespace_path,
)
)
python/python/lancedb/storage_integration.py
+3 -1
View File
@@ -30,7 +30,7 @@ class StorageIntegrationInfo:
name: str
database: str
namespace_path: list[str]
storage_provider: Literal["S3", "AZURE"]
storage_provider: Literal["S3", "AZURE", "GCP"]
storage_prefix: str
aws_region: Optional[str]
aws_account_id: str
@@ -48,6 +48,7 @@ class StorageIntegrationInfo:
azure_tenant_id_secret: Optional[SecretReference] = None
azure_client_id_secret: Optional[SecretReference] = None
azure_client_secret_secret: Optional[SecretReference] = None
gcp_credentials_json_secret: Optional[SecretReference] = None
def __post_init__(self):
for field in (
@@ -56,6 +57,7 @@ class StorageIntegrationInfo:
"azure_tenant_id_secret",
"azure_client_id_secret",
"azure_client_secret_secret",
"gcp_credentials_json_secret",
):
value = getattr(self, field)
if isinstance(value, dict):
python/python/tests/test_storage_integration.py
+87
View File
@@ -43,6 +43,25 @@ def remote_server(text_type=pa.string()):
elif b"DROP STORAGE INTEGRATION" in command:
result = pa.table({"dropped": [True]})
else:
if b"GCP_CREDENTIALS_JSON_SECRET" in command:
info.update(
storage_provider="GCP",
storage_prefix="gs://customer-data/raw/",
aws_role_arn=None,
aws_region=None,
aws_account_id="",
aws_iam_principal_arn="",
trust_policy=None,
permissions_policy=None,
configuration_instructions=(
"Grant the referenced identity access to the prefix."
),
gcp_credentials_json_secret=dict(
database="shared",
namespace_path=["credentials"],
name="gcp",
),
)
if b"STORAGE_AZURE_STORAGE_ACCOUNT" in command:
info.update(
storage_provider="AZURE",
@@ -83,6 +102,7 @@ def remote_server(text_type=pa.string()):
"azure_tenant_id_secret",
"azure_client_id_secret",
"azure_client_secret_secret",
"gcp_credentials_json_secret",
]:
if field in row and row[field] is not None:
row[field] = json.dumps(row[field])
@@ -305,3 +325,70 @@ async def test_async_azure_requires_account_and_all_three_secret_references():
)
assert info.storage_provider == "AZURE"
assert b'STORAGE_AZURE_TENANT_ID_SECRET = "public"."tenant"' in calls[0]
@pytest.mark.parametrize(
"text_type", [pa.string(), pa.large_string(), pa.string_view()]
)
def test_gcp_lifecycle_uses_one_qualified_json_secret(text_type):
with remote_server(text_type) as (host, calls):
db = connect(host)
info = db.create_storage_integration(
"gcs",
"gs://customer-data/raw/",
storage_provider="GCP",
gcp_credentials_json_secret=SecretReference(
"gcp", database="shared", namespace_path=("credentials",)
),
)
assert info.storage_provider == "GCP"
assert info.gcp_credentials_json_secret.name == "gcp"
assert info.gcp_credentials_json_secret.database == "shared"
assert info.aws_role_arn is None
assert info.azure_storage_account is None
assert info.aws_iam_principal_arn == ""
assert info.trust_policy is None
assert not info.access_verified
assert db.describe_storage_integration("gcs") == info
assert list(db.list_storage_integrations()) == ["raw", "zeta"]
db.drop_storage_integration("gcs")
assert b"STORAGE_PROVIDER = 'GCP'" in calls[0]
assert b'GCP_CREDENTIALS_JSON_SECRET = "shared"."credentials"."gcp"' in calls[0]
assert b"STORAGE_AWS" not in calls[0]
assert b"STORAGE_AZURE" not in calls[0]
@pytest.mark.asyncio
async def test_async_gcp_requires_one_secret_and_rejects_other_cloud_options():
with remote_server() as (host, calls):
db = await lancedb.connect_async(
"db://default",
api_key="fake",
sql_host_override=host,
host_override="http://127.0.0.1:1",
)
for invalid in (
{},
{"gcp_credentials_json_secret": "gcp", "aws_role_arn": ROLE},
{"gcp_credentials_json_secret": "gcp", "azure_storage_account": "account"},
):
with pytest.raises(ValueError):
await db.create_storage_integration(
"gcs", "gs://customer-data/raw/", storage_provider="GCP", **invalid
)
with pytest.raises(ValueError):
await db.create_storage_integration(
"gcs",
"gs://customer-data/raw/",
aws_role_arn=ROLE,
gcp_credentials_json_secret="gcp",
)
assert calls == []
info = await db.create_storage_integration(
"gcs",
"gs://customer-data/raw/",
storage_provider="GCP",
gcp_credentials_json_secret="gcp",
)
assert info.storage_provider == "GCP"
assert b'GCP_CREDENTIALS_JSON_SECRET = "public"."gcp"' in calls[0]