diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 27b967ac4e..06ac4338b6 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -8574,6 +8574,7 @@ dependencies = [ "rand 0.9.0", "reqwest 0.12.20", "ring 0.17.14", + "rustls-pemfile 2.2.0", "serde", "serde_json", "serde_urlencoded", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 3aebd08d16..2eb24ebfb6 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -348,7 +348,7 @@ nkeys = "0.4.4" nu-parser = { version = "0.101.0", default-features = false } datafusion = "47.0.0" -object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure"] } +object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" aws-sdk-sqs = "1.57.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 3afeb766d1..188513dd18 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -651b945d4567081968005278d7e87ea41cabbe1c +b7c6fc065a3da98933d50fae697784da61c3fe2d diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8f6f925f65..3e315b8d21 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -16563,11 +16563,13 @@ components: properties: type: type: string - enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"] + enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"] s3_resource_path: type: string azure_blob_resource_path: type: string + gcs_resource_path: + type: string public_resource: type: boolean secondary_storage: @@ -16578,11 +16580,13 @@ components: type: type: string enum: - ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"] + ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"] s3_resource_path: type: string azure_blob_resource_path: type: string + gcs_resource_path: + type: string public_resource: type: boolean diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 171958f5cd..f838e2ef48 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -10,6 +10,8 @@ use object_store::aws::AwsCredential; #[cfg(feature = "parquet")] use object_store::azure::MicrosoftAzureBuilder; #[cfg(feature = "parquet")] +use object_store::gcp::GoogleCloudStorageBuilder; +#[cfg(feature = "parquet")] use object_store::ObjectStore; #[cfg(feature = "parquet")] use object_store::{aws::AmazonS3Builder, ClientOptions}; @@ -221,6 +223,7 @@ pub enum LargeFileStorage { AzureBlobStorage(AzureBlobStorage), S3AwsOidc(S3Storage), AzureWorkloadIdentity(AzureBlobStorage), + GoogleCloudStorage(GoogleCloudStorage), // TODO: Add a filesystem type here in the future if needed } @@ -238,10 +241,18 @@ pub struct AzureBlobStorage { pub public_resource: Option, } +#[derive(Serialize, Deserialize, Debug)] +pub struct GoogleCloudStorage { + pub gcs_resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_resource: Option, +} + #[derive(Clone, Debug)] pub enum ObjectStoreResource { S3(S3Resource), Azure(AzureBlobResource), + Gcs(GcsResource), } impl ObjectStoreResource { @@ -259,6 +270,7 @@ pub enum StorageResourceType { AzureBlob, S3AwsOidc, AzureWorkloadIdentity, + GoogleCloudStorage, } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -300,6 +312,22 @@ pub struct AzureBlobResource { pub federated_token_file: Option, } +fn as_string<'de, D>(deserializer: D) -> Result +where + D: serde::de::Deserializer<'de>, +{ + let v: serde_json::Value = Deserialize::deserialize(deserializer)?; + serde_json::to_string(&v).map_err(serde::de::Error::custom) +} + +#[derive(Debug, Deserialize, Clone)] +pub struct GcsResource { + pub bucket: String, + #[serde(rename = "serviceAccountKey")] + #[serde(deserialize_with = "as_string")] + pub service_account_key: String, +} + #[derive(Debug, Deserialize, Serialize, Clone, Hash)] pub struct S3AwsOidcResource { #[serde(rename = "bucket")] @@ -380,6 +408,7 @@ pub async fn build_object_store_client( ObjectStoreResource::Azure(azure_blob_resource_ref) => { build_azure_blob_client(&azure_blob_resource_ref) } + ObjectStoreResource::Gcs(gcs_resource_ref) => build_gcs_client(&gcs_resource_ref).await, } } @@ -575,18 +604,59 @@ fn build_azure_blob_client( return Ok(Arc::new(store)); } +#[cfg(feature = "parquet")] +async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result> { + let gcs_resource = gcs_resource_ref.clone(); + + let mut store_builder = GoogleCloudStorageBuilder::new() + .with_client_options( + ClientOptions::new() + .with_timeout_disabled() + .with_default_headers(HeaderMap::from_iter(vec![( + "Accept-Encoding".parse().unwrap(), + "".parse().unwrap(), + )])), + ) + .with_bucket_name(gcs_resource.bucket); + + store_builder = store_builder.with_service_account_key(gcs_resource.service_account_key); + + // if private key is malformed, it will panic => https://github.com/apache/arrow-rs-object-store/issues/419 + let store = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store_builder.build())) + .map_err(|panic_info| { + tracing::error!( + "Panic while building GCS object store client: {:?}", + panic_info + ); + error::Error::internal_err(format!( + "Panic while building GCS object store client: {:?}", + panic_info + )) + })? + .map_err(|err| { + tracing::error!("Error building GCS object store client: {:?}", err); + error::Error::internal_err(format!( + "Error building GCS object store client: {}", + err.to_string() + )) + })?; + + return Ok(Arc::new(store)); +} + #[derive(Serialize, Deserialize)] #[serde(tag = "typ", content = "value")] pub enum ObjectStoreSettings { S3(S3Settings), } -#[derive(Debug, Deserialize, Serialize, Clone)] +#[derive(Debug, Deserialize, Clone)] #[serde(tag = "type")] pub enum ObjectSettings { S3(S3Settings), Azure(AzureBlobResource), AwsOidc(S3AwsOidcResource), + Gcs(GcsResource), } impl ObjectSettings { @@ -595,6 +665,7 @@ impl ObjectSettings { ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(), ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name), ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket), + ObjectSettings::Gcs(gcs_settings) => Some(&gcs_settings.bucket), } } } @@ -628,6 +699,12 @@ pub async fn build_object_store_from_settings( refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())), }) } + ObjectSettings::Gcs(gcs_settings) => { + let gcs_resource = gcs_settings; + build_gcs_client(&gcs_resource) + .await + .map(|x| ExpirableObjectStore::from(x)) + } } } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 3b5312c14c..7008382c6b 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -775,6 +775,13 @@ async fn get_workspace_s3_resource_path( resource_path.to_string(), ) } + Some(LargeFileStorage::GoogleCloudStorage(gcs)) => { + let resource_path = gcs.gcs_resource_path.trim_start_matches("$res:"); + ( + StorageResourceType::GoogleCloudStorage, + resource_path.to_string(), + ) + } None => { return Ok(None); } diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 58bdee82c3..51cbe3452f 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -649,7 +649,7 @@