starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 16:05:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: add support for GCS object storage (#6083)

Browse Source 
* AI: Updates to files (run 15923369101)

* all

* ee ref

* unneeded

---------

Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: HugoCasa <hugo@casademont.ch>
This commit is contained in:
windmill-internal-app[bot]
2025-06-30 20:11:57 +00:00
committed by GitHub
co-authored by windmill-internal-app[bot] HugoCasa
parent 8ba3959ada
commit c51e128920
10 changed files with 174 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+1
View File
@@ -8574,6 +8574,7 @@ dependencies = [
"rand 0.9.0",
"reqwest 0.12.20",
"ring 0.17.14",
"rustls-pemfile 2.2.0",
"serde",
"serde_json",
"serde_urlencoded",
backend/Cargo.toml
+1 -1
View File
@@ -348,7 +348,7 @@ nkeys = "0.4.4"
nu-parser = { version = "0.101.0", default-features = false }
datafusion = "47.0.0"
object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure"] }
object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] }
openidconnect = { version = "4.0.0-rc.1" }
aws-config = "^1"
aws-sdk-sqs = "1.57.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
651b945d4567081968005278d7e87ea41cabbe1c
b7c6fc065a3da98933d50fae697784da61c3fe2d
backend/windmill-api/openapi.yaml
+6 -2
View File
@@ -16563,11 +16563,13 @@ components:
properties:
type:
type: string
enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"]
enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"]
s3_resource_path:
type: string
azure_blob_resource_path:
type: string
gcs_resource_path:
type: string
public_resource:
type: boolean
secondary_storage:
@@ -16578,11 +16580,13 @@ components:
type:
type: string
enum:
["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"]
["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc", "GoogleCloudStorage"]
s3_resource_path:
type: string
azure_blob_resource_path:
type: string
gcs_resource_path:
type: string
public_resource:
type: boolean
backend/windmill-common/src/s3_helpers.rs
+78 -1
View File
@@ -10,6 +10,8 @@ use object_store::aws::AwsCredential;
#[cfg(feature = "parquet")]
use object_store::azure::MicrosoftAzureBuilder;
#[cfg(feature = "parquet")]
use object_store::gcp::GoogleCloudStorageBuilder;
#[cfg(feature = "parquet")]
use object_store::ObjectStore;
#[cfg(feature = "parquet")]
use object_store::{aws::AmazonS3Builder, ClientOptions};
@@ -221,6 +223,7 @@ pub enum LargeFileStorage {
AzureBlobStorage(AzureBlobStorage),
S3AwsOidc(S3Storage),
AzureWorkloadIdentity(AzureBlobStorage),
GoogleCloudStorage(GoogleCloudStorage),
// TODO: Add a filesystem type here in the future if needed
}
@@ -238,10 +241,18 @@ pub struct AzureBlobStorage {
pub public_resource: Option<bool>,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct GoogleCloudStorage {
pub gcs_resource_path: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub public_resource: Option<bool>,
}
#[derive(Clone, Debug)]
pub enum ObjectStoreResource {
S3(S3Resource),
Azure(AzureBlobResource),
Gcs(GcsResource),
}
impl ObjectStoreResource {
@@ -259,6 +270,7 @@ pub enum StorageResourceType {
AzureBlob,
S3AwsOidc,
AzureWorkloadIdentity,
GoogleCloudStorage,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
@@ -300,6 +312,22 @@ pub struct AzureBlobResource {
pub federated_token_file: Option<String>,
}
fn as_string<'de, D>(deserializer: D) -> Result<String, D::Error>
where
D: serde::de::Deserializer<'de>,
{
let v: serde_json::Value = Deserialize::deserialize(deserializer)?;
serde_json::to_string(&v).map_err(serde::de::Error::custom)
}
#[derive(Debug, Deserialize, Clone)]
pub struct GcsResource {
pub bucket: String,
#[serde(rename = "serviceAccountKey")]
#[serde(deserialize_with = "as_string")]
pub service_account_key: String,
}
#[derive(Debug, Deserialize, Serialize, Clone, Hash)]
pub struct S3AwsOidcResource {
#[serde(rename = "bucket")]
@@ -380,6 +408,7 @@ pub async fn build_object_store_client(
ObjectStoreResource::Azure(azure_blob_resource_ref) => {
build_azure_blob_client(&azure_blob_resource_ref)
}
ObjectStoreResource::Gcs(gcs_resource_ref) => build_gcs_client(&gcs_resource_ref).await,
}
}
@@ -575,18 +604,59 @@ fn build_azure_blob_client(
return Ok(Arc::new(store));
}
#[cfg(feature = "parquet")]
async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result<Arc<dyn ObjectStore>> {
let gcs_resource = gcs_resource_ref.clone();
let mut store_builder = GoogleCloudStorageBuilder::new()
.with_client_options(
ClientOptions::new()
.with_timeout_disabled()
.with_default_headers(HeaderMap::from_iter(vec![(
"Accept-Encoding".parse().unwrap(),
"".parse().unwrap(),
)])),
)
.with_bucket_name(gcs_resource.bucket);
store_builder = store_builder.with_service_account_key(gcs_resource.service_account_key);
// if private key is malformed, it will panic => https://github.com/apache/arrow-rs-object-store/issues/419
let store = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store_builder.build()))
.map_err(|panic_info| {
tracing::error!(
"Panic while building GCS object store client: {:?}",
panic_info
);
error::Error::internal_err(format!(
"Panic while building GCS object store client: {:?}",
panic_info
))
})?
.map_err(|err| {
tracing::error!("Error building GCS object store client: {:?}", err);
error::Error::internal_err(format!(
"Error building GCS object store client: {}",
err.to_string()
))
})?;
return Ok(Arc::new(store));
}
#[derive(Serialize, Deserialize)]
#[serde(tag = "typ", content = "value")]
pub enum ObjectStoreSettings {
S3(S3Settings),
}
#[derive(Debug, Deserialize, Serialize, Clone)]
#[derive(Debug, Deserialize, Clone)]
#[serde(tag = "type")]
pub enum ObjectSettings {
S3(S3Settings),
Azure(AzureBlobResource),
AwsOidc(S3AwsOidcResource),
Gcs(GcsResource),
}
impl ObjectSettings {
@@ -595,6 +665,7 @@ impl ObjectSettings {
ObjectSettings::S3(s3_settings) => s3_settings.bucket.as_ref(),
ObjectSettings::Azure(azure_settings) => Some(&azure_settings.container_name),
ObjectSettings::AwsOidc(s3_aws_oidc_settings) => Some(&s3_aws_oidc_settings.bucket),
ObjectSettings::Gcs(gcs_settings) => Some(&gcs_settings.bucket),
}
}
}
@@ -628,6 +699,12 @@ pub async fn build_object_store_from_settings(
refresh: Some(ObjectStoreRefresh::new(settings.clone(), res.expiration())),
})
}
ObjectSettings::Gcs(gcs_settings) => {
let gcs_resource = gcs_settings;
build_gcs_client(&gcs_resource)
.await
.map(|x| ExpirableObjectStore::from(x))
}
}
}
backend/windmill-worker/src/common.rs
+7
View File
@@ -775,6 +775,13 @@ async fn get_workspace_s3_resource_path(
resource_path.to_string(),
)
}
Some(LargeFileStorage::GoogleCloudStorage(gcs)) => {
let resource_path = gcs.gcs_resource_path.trim_start_matches("$res:");
(
StorageResourceType::GoogleCloudStorage,
resource_path.to_string(),
)
}
None => {
return Ok(None);
}
frontend/src/lib/components/InstanceSetting.svelte
+2 -2
View File
@@ -649,7 +649,7 @@
<Button
variant="border"
color="light"
size="md"
size="xs"
btnClasses="mt-1"
on:click={() => {
if ($values[setting.key] == undefined || !Array.isArray($values[setting.key])) {
@@ -668,7 +668,7 @@
disabled={!$enterpriseLicense}
variant="border"
color="light"
size="md"
size="xs"
on:click={async () => {
try {
await SettingService.testCriticalChannels({
frontend/src/lib/components/ObjectStoreConfigSettings.svelte
+57 -4
View File
@@ -6,6 +6,7 @@
import { sendUserToast } from '$lib/toast'
import TestConnection from './TestConnection.svelte'
import { enterpriseLicense } from '$lib/stores'
import SimpleEditor from './SimpleEditor.svelte'
type S3Config = {
type: 'S3'
@@ -35,7 +36,14 @@
roleArn: string
}
export let bucket_config: S3Config | AzureConfig | AwsOidcConfig | undefined = undefined
type GcsConfig = {
type: 'Gcs'
bucket: string
serviceAccountKey: Record<string, string>
}
export let bucket_config: S3Config | AzureConfig | AwsOidcConfig | GcsConfig | undefined =
undefined
$: bucket_config?.type == 'S3' &&
bucket_config.allow_http == undefined &&
@@ -107,9 +115,9 @@
</div>
<Tabs
bind:selected={bucket_config.type}
selected={bucket_config?.type ?? 'S3'}
on:selected={(e) => {
if (e.detail === 'S3') {
if (e.detail === 'S3' && bucket_config?.type !== 'S3') {
bucket_config = {
type: 'S3',
bucket: '',
@@ -118,7 +126,7 @@
secret_key: '',
endpoint: ''
}
} else if (e.detail === 'Azure') {
} else if (e.detail === 'Azure' && bucket_config?.type !== 'Azure') {
bucket_config = {
type: 'Azure',
accountName: '',
@@ -128,12 +136,26 @@
clientId: '',
accessKey: ''
}
} else if (e.detail === 'Gcs' && bucket_config?.type !== 'Gcs') {
bucket_config = {
type: 'Gcs',
bucket: '',
serviceAccountKey: {}
}
} else if (e.detail === 'AwsOidc' && bucket_config?.type !== 'AwsOidc') {
bucket_config = {
type: 'AwsOidc',
bucket: '',
region: '',
roleArn: ''
}
}
}}
>
<Tab size="sm" value="S3">S3</Tab>
<Tab size="sm" value="Azure">Azure Blob</Tab>
<Tab size="sm" value="AwsOidc">AWS OIDC</Tab>
<Tab size="sm" value="Gcs">Google Cloud Storage</Tab>
</Tabs>
<div class="flex flex-col gap-2 mt-2 p-2 border rounded-md">
{#if bucket_config.type === 'S3'}
@@ -236,6 +258,37 @@
bind:value={bucket_config.roleArn}
/>
</label>
{:else if bucket_config.type === 'Gcs'}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Bucket</span>
<input type="text" placeholder="bucket-name" bind:value={bucket_config.bucket} />
</label>
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Service Account Key</span>
<span class="text-tertiary text-2xs">JSON content of the service account key file</span>
<SimpleEditor
lang="json"
bind:code={
() => {
if (bucket_config?.type === 'Gcs') {
return JSON.stringify(bucket_config.serviceAccountKey)
} else {
return '{}'
}
},
(v) => {
if (bucket_config?.type === 'Gcs') {
try {
bucket_config.serviceAccountKey = JSON.parse(v ?? '{}')
} catch (_) {
bucket_config.serviceAccountKey = {}
}
}
}
}
class="h-80"
/>
</label>
{:else}
<div>Unknown bucket type {bucket_config['type']}</div>
{/if}
frontend/src/lib/components/workspaceSettings/StorageSettings.svelte
+7 -3
View File
@@ -39,12 +39,15 @@
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class="text-primary text-lg font-semibold">Workspace Object Storage (S3/Azure Blob)</div>
<div class="text-primary text-lg font-semibold"
>Workspace Object Storage (S3/Azure Blob/GCS)</div
>
<Description
link="https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill#workspace-object-storage"
>
Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable users
to read and write from S3 without having to have access to the credentials.
Connect your Windmill workspace to your S3 bucket, Azure Blob storage, or Google Cloud Storage
to enable users to read and write from object storage without having to have access to the
credentials.
</Description>
</div>
</div>
@@ -74,6 +77,7 @@
<Tab size="xs" value="azure_blob">Azure Blob</Tab>
<Tab exact size="xs" value="s3_aws_oidc">AWS OIDC</Tab>
<Tab size="xs" value="azure_workload_identity">Azure Workload Identity</Tab>
<Tab exact size="xs" value="gcloud_storage">Google Cloud Storage</Tab>
</Tabs>
</div>
<div class="w-full flex gap-1 mt-4">
frontend/src/lib/workspace_settings.ts
+14 -1
View File
@@ -1,7 +1,9 @@
import type { GetSettingsResponse, LargeFileStorage } from './gen'
import { emptyString } from './utils'
type s3type = 's3' | 'azure_blob' | 's3_aws_oidc' | 'azure_workload_identity'
// Extended type to include GCS support until backend types are regenerated
type s3type = 's3' | 'azure_blob' | 's3_aws_oidc' | 'azure_workload_identity' | 'gcloud_storage'
type s3ResourceSettingsItem = {
resourceType: s3type
resourcePath: string | undefined
@@ -49,6 +51,13 @@ export function convertBackendSettingsToFrontendSettingsItem(
resourcePath: large_file_storage?.s3_resource_path?.replace('$res:', ''),
publicResource: large_file_storage?.public_resource
}
} else if (large_file_storage?.type === 'GoogleCloudStorage') {
const gcsStorage = large_file_storage
return {
resourceType: 'gcloud_storage',
resourcePath: gcsStorage?.gcs_resource_path?.replace('$res:', ''),
publicResource: gcsStorage?.public_resource
}
} else {
return {
resourceType: 's3',
@@ -91,6 +100,10 @@ export function convertFrontendToBackendettingsItem(
let typ: LargeFileStorage['type'] = 'S3AwsOidc'
params['type'] = typ
params['s3_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 'gcloud_storage') {
let typ: LargeFileStorage['type'] = 'GoogleCloudStorage'
params['type'] = typ
params['gcs_resource_path'] = resourcePathWithPrefix
} else {
let typ: LargeFileStorage['type'] = 'S3Storage'
params['type'] = typ