From 4578ed32da42b154833c6307cc6f82b526b05ab4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 8 Mar 2024 22:51:03 +0100 Subject: [PATCH] feat: implement s3 oidc support + azure workload identity support --- backend/Cargo.lock | 364 +++++++++++++++++- backend/Cargo.toml | 2 + backend/ee-repo-ref.txt | 2 +- backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/openapi.yaml | 2 +- backend/windmill-common/Cargo.toml | 4 +- backend/windmill-common/src/error.rs | 2 +- .../windmill-common/src/job_s3_helpers_ee.rs | 1 + backend/windmill-common/src/lib.rs | 3 + backend/windmill-common/src/s3_helpers.rs | 95 ++++- backend/windmill-worker/src/common.rs | 77 ++-- backend/windmill-worker/src/worker.rs | 18 +- cli/metadata.ts | 10 +- .../apps/components/buttons/AppButton.svelte | 10 - .../src/lib/components/common/tabs/Tab.svelte | 12 +- .../(root)/(logged)/resources/+page.svelte | 6 +- .../(logged)/workspace_settings/+page.svelte | 273 ++++++++----- 17 files changed, 694 insertions(+), 188 deletions(-) create mode 120000 backend/windmill-common/src/job_s3_helpers_ee.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 000b246d80..68dac532ea 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -398,7 +398,7 @@ checksum = "beec3f8fb8f710b7be84ccd1716e17f38f2868168355cab5f2f168ae988e767e" dependencies = [ "base64 0.21.7", "bytes", - "http", + "http 0.2.12", "rand 0.8.5", "reqwest", "serde", @@ -575,6 +575,300 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b96342ea8948ab9bef3e6234ea97fc32e2d8a88d8fb6a084e52267317f94b6b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.0.1", + "hex", + "http 0.2.12", + "hyper", + "ring 0.17.8", + "time", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273fa47dafc9ef14c2c074ddddbea4561ff01b7f68d5091c0e9737ced605c01d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e38bab716c8bf07da24be07ecc02e0f5656ce8f30a891322ecdcb202f943b85" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand 2.0.1", + "http 0.2.12", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid 1.7.0", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d84bd3925a17c9adbf6ec65d52104a44a09629d8f70290542beeee69a95aee7f" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c2dae39e997f58bc4d6292e6244b26ba630c01ab671b6f9f44309de3eb80ab8" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17fd9a53869fee17cea77e352084e1aa71e2c5e323d974c13a9c2bcfd9544c7f" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ada00a4645d7d89f296fe0ddbc3fe3554f03035937c849a05d37ddffc1f29a1" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "percent-encoding", + "sha2 0.10.8", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf7f09a27286d84315dfb9346208abb3b0973a692454ae6d0bc8d803fcce3b4" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ca214a6a26f1b7ebd63aa8d4f5e2194095643023f9608edf99a58247b9d80d" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1af80ecf3057fb25fe38d1687e94c4601a7817c6a1e87c1b0635f7ecb644ace5" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb27084f72ea5fc20033efe180618677ff4a2f474b53d84695cfe310a6526cbc" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5fca54a532a36ff927fbd7407a7c8eb9c3b4faf72792ba2965ea2cad8ed55" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand 2.0.1", + "h2", + "http 0.2.12", + "http-body", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.10", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22389cb6f7cac64f266fb9f137745a9349ced7b47e0d2ba503e9e40ede4f7060" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f081da5481210523d44ffd83d9f0740320050054006c719eae0232d411f024d3" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fccd8f595d0ca839f9f2548e66b99514a85f92feb4c01cf2868d93eb4888a42" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07c63521aa1ea9a9f92a701f1a08ce3fd20b46c6efc0d5c8947c1fd879e3df1" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 0.2.12", + "rustc_version 0.4.0", + "tracing", +] + [[package]] name = "axum" version = "0.6.20" @@ -587,7 +881,7 @@ dependencies = [ "bytes", "futures-util", "headers", - "http", + "http 0.2.12", "http-body", "hyper", "itoa", @@ -617,7 +911,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", + "http 0.2.12", "http-body", "mime", "rustversion", @@ -978,6 +1272,16 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -1759,7 +2063,7 @@ dependencies = [ "deno_core", "deno_tls", "dyn-clone", - "http", + "http 0.2.12", "pin-project", "reqwest", "serde", @@ -3002,7 +3306,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap 2.2.5", "slab", "tokio", @@ -3071,7 +3375,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http", + "http 0.2.12", "httpdate", "mime", "sha1", @@ -3083,7 +3387,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.12", ] [[package]] @@ -3175,6 +3479,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -3182,7 +3497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", "pin-project-lite", ] @@ -3202,7 +3517,7 @@ dependencies = [ "async-channel 1.9.0", "base64 0.13.1", "futures-lite 1.13.0", - "http", + "http 0.2.12", "infer", "pin-project-lite", "rand 0.7.3", @@ -3242,7 +3557,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.12", "http-body", "httparse", "httpdate", @@ -3262,8 +3577,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", + "http 0.2.12", "hyper", + "log", "rustls 0.21.10", "rustls-native-certs", "tokio", @@ -4251,7 +4567,7 @@ dependencies = [ "base64 0.13.1", "chrono", "getrandom 0.2.12", - "http", + "http 0.2.12", "rand 0.8.5", "reqwest", "serde", @@ -4356,7 +4672,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac", - "http", + "http 0.2.12", "itertools 0.10.5", "log", "oauth2", @@ -5375,7 +5691,7 @@ source = "git+https://github.com/oxidecomputer/progenitor?rev=3d96016ae8d422e905 dependencies = [ "getopts", "heck", - "http", + "http 0.2.12", "indexmap 1.9.3", "openapiv3", "proc-macro2", @@ -5763,6 +6079,12 @@ dependencies = [ "regex-syntax 0.8.2", ] +[[package]] +name = "regex-lite" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -5813,7 +6135,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", + "http 0.2.12", "http-body", "hyper", "hyper-rustls", @@ -8241,7 +8563,7 @@ dependencies = [ "axum-core", "cookie", "futures-util", - "http", + "http 0.2.12", "parking_lot", "pin-project-lite", "tower-layer", @@ -8258,7 +8580,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", + "http 0.2.12", "http-body", "http-range-header", "pin-project-lite", @@ -9199,6 +9521,8 @@ name = "windmill-common" version = "1.287.1" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-sts", "axum", "chrono", "cron", @@ -9646,6 +9970,12 @@ dependencies = [ "rustix 0.38.31", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xxhash-rust" version = "0.8.10" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5fe9ae25aa..cb15efe7f8 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -227,3 +227,5 @@ polars-io = { version = "0.35.4", features = ["csv"] } object_store = { version = "0.8.0", features = ["aws", "azure"] } openidconnect = { version = "3.4.0" } +aws-config = "^1" +aws-sdk-sts = "^1" \ No newline at end of file diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 140fdd7904..70ed580071 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -7733647d5d3026654d3dcc279ec0cae934532f8a \ No newline at end of file +e419b8dad3a83783b4faaacd2396245c0d776f92 \ No newline at end of file diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 73ffede87c..2556d1ca9e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -87,3 +87,4 @@ polars-io = { workspace = true, optional = true} object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} pin-project.workspace = true + diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1695f7bf33..f009bf040a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9673,7 +9673,7 @@ components: properties: type: type: string - enum: ["S3Storage", "AzureBlobStorage"] + enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"] s3_resource_path: type: string azure_blob_resource_path: diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 65584d717f..3cfbc6092d 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -10,7 +10,7 @@ enterprise = [] prometheus = ["dep:prometheus"] flamegraph = ["dep:tracing-flame"] loki = ["dep:tracing-loki"] -parquet = ["dep:object_store"] +parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"] [lib] name = "windmill_common" @@ -44,3 +44,5 @@ tracing-loki = { version = "^0", optional = true } magic-crypt.workspace = true object_store = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-sdk-sts = { workspace = true, optional = true } \ No newline at end of file diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 6cf0a194cc..e2321251a6 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -90,7 +90,7 @@ impl IntoResponse for Error { if matches!(status, axum::http::StatusCode::NOT_FOUND) { tracing::warn!(message = e.to_string()); } else { - tracing::error!(message = e.to_string()); + tracing::error!(message = e.to_string(), error = ?e); }; axum::response::Response::builder() diff --git a/backend/windmill-common/src/job_s3_helpers_ee.rs b/backend/windmill-common/src/job_s3_helpers_ee.rs new file mode 120000 index 0000000000..6c8430be70 --- /dev/null +++ b/backend/windmill-common/src/job_s3_helpers_ee.rs @@ -0,0 +1 @@ +/git/windmill/../windmill-ee-private/windmill-common/src/job_s3_helpers_ee.rs \ No newline at end of file diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 23fb3aa628..913b9446bd 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -24,10 +24,13 @@ pub mod flow_status; pub mod flows; pub mod global_settings; pub mod job_metrics; +#[cfg(feature = "parquet")] +pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; pub mod s3_helpers; + pub mod schedule; pub mod scripts; pub mod server; diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 2b3c49af35..2ea94fc604 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -15,6 +15,8 @@ use std::sync::Arc; pub enum LargeFileStorage { S3Storage(S3Storage), AzureBlobStorage(AzureBlobStorage), + S3AwsOidc(S3Storage), + AzureWorkloadIdentity(AzureBlobStorage), // TODO: Add a filesystem type here in the future if needed } @@ -38,6 +40,14 @@ pub enum ObjectStoreResource { AzureBlobResource(AzureBlobResource), } +#[derive(Deserialize, Debug)] +pub enum StorageResourceType { + S3, + AzureBlob, + S3AwsOidc, + AzureWorkloadIdentity, +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct S3Resource { #[serde(rename = "bucket")] @@ -52,7 +62,8 @@ pub struct S3Resource { #[serde(rename = "secretKey")] pub secret_key: Option, #[serde(rename = "pathStyle")] - pub path_style: bool, + pub path_style: Option, + pub token: Option, pub port: Option, } @@ -61,13 +72,29 @@ pub struct AzureBlobResource { #[serde(rename = "endpoint")] pub endpoint: Option, #[serde(rename = "useSSL")] - pub use_ssl: bool, + pub use_ssl: Option, #[serde(rename = "accountName")] pub account_name: String, + #[serde(rename = "tenantId")] + pub tenant_id: Option, + #[serde(rename = "clientId")] + pub client_id: Option, #[serde(rename = "containerName")] pub container_name: String, #[serde(rename = "accessKey")] pub access_key: Option, + #[serde(rename = "federatedTokenFile")] + pub federated_token_file: Option, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct S3AwsOidcResource { + #[serde(rename = "bucket")] + pub bucket: String, + pub region: Option, + #[serde(rename = "roleArn")] + pub role_arn: String, + pub audience: Option, } #[derive(Deserialize, Clone)] @@ -96,14 +123,27 @@ pub async fn get_etag_or_empty( .flatten(); } -pub fn render_endpoint(raw_endpoint: String, use_ssl: bool, port: Option) -> String { +pub fn render_endpoint( + raw_endpoint: String, + use_ssl: bool, + port: Option, + path_style: Option, + bucket: String, +) -> String { let url_with_prefix = if raw_endpoint.starts_with("http://") || raw_endpoint.starts_with("https://") { raw_endpoint.clone() - } else if use_ssl { - format!("https://{}", raw_endpoint) } else { - format!("http://{}", raw_endpoint) + let scheme = if use_ssl { "https" } else { "http" }; + format!( + "{}://{}", + scheme, + if path_style.unwrap_or(true) { + raw_endpoint + } else { + format!("{}.{}", bucket, raw_endpoint) + } + ) }; if port.is_some() { format!("{}:{}", url_with_prefix, port.unwrap()) @@ -127,7 +167,13 @@ pub fn build_object_store_client( #[cfg(feature = "parquet")] fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result> { let s3_resource = s3_resource_ref.clone(); - let endpoint = render_endpoint(s3_resource.endpoint, s3_resource.use_ssl, s3_resource.port); + let endpoint = render_endpoint( + s3_resource.endpoint, + s3_resource.use_ssl, + s3_resource.port, + s3_resource.path_style, + s3_resource.bucket.clone(), + ); let mut store_builder = AmazonS3Builder::new() .with_client_options(ClientOptions::new().with_timeout_disabled()) // TODO: make it configurable maybe @@ -144,13 +190,19 @@ fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result Option { +) -> windmill_common::error::Result> { + use windmill_common::{ + job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType, + }; + let raw_lfs_opt = sqlx::query_scalar!( "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", workspace_id @@ -1060,33 +1061,46 @@ async fn get_workspace_s3_resource_path( .map(|val| serde_json::from_value::(val).ok()) .flatten(); - match raw_lfs_opt { + let (rt, path) = match raw_lfs_opt { Some(LargeFileStorage::S3Storage(s3_storage)) => { let resource_path = s3_storage.s3_resource_path.trim_start_matches("$res:"); - let s3_resource = client - .get_resource_value_interpolated::( - &resource_path, - Some(job_id.to_string()), - ) - .await - .ok(); - s3_resource.map(|resource| ObjectStoreResource::S3Resource(resource)) + (StorageResourceType::S3, resource_path.to_string()) } Some(LargeFileStorage::AzureBlobStorage(azure_blob_storage)) => { let resource_path = azure_blob_storage .azure_blob_resource_path .trim_start_matches("$res:"); - let azure_blob_resource = client - .get_resource_value_interpolated::( - &resource_path, - Some(job_id.to_string()), - ) - .await - .ok(); - azure_blob_resource.map(|resource| ObjectStoreResource::AzureBlobResource(resource)) + (StorageResourceType::AzureBlob, resource_path.to_string()) } - None => None, - } + Some(LargeFileStorage::S3AwsOidc(s3_aws_oidc)) => { + let resource_path = s3_aws_oidc.s3_resource_path.trim_start_matches("$res:"); + (StorageResourceType::S3AwsOidc, resource_path.to_string()) + } + Some(LargeFileStorage::AzureWorkloadIdentity(azure)) => { + let resource_path = azure.azure_blob_resource_path.trim_start_matches("$res:"); + ( + StorageResourceType::AzureWorkloadIdentity, + resource_path.to_string(), + ) + } + None => { + return Ok(None); + } + }; + + let client2 = client.clone(); + let token_fn = |audience: String| async move { + client2 + .get_id_token(&audience) + .await + .map_err(|e| windmill_common::error::Error::from(e)) + }; + let s3_resource_value_raw = client + .get_resource_value::(path.as_str()) + .await?; + get_s3_resource_internal(rt, s3_resource_value_raw, token_fn) + .await + .map(Some) } #[cfg(feature = "parquet")] @@ -1094,7 +1108,6 @@ async fn arg_value_hash_additions( db: &DB, client: &AuthedClient, workspace_id: &str, - job_id: &Uuid, raw_value: &Box, ) -> HashMap { let mut result: HashMap = HashMap::new(); @@ -1105,8 +1118,8 @@ async fn arg_value_hash_additions( return result; } - let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id, job_id).await; - if let Some(s3_resource) = s3_resource_opt { + let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id).await; + if let Some(s3_resource) = s3_resource_opt.ok().flatten() { for s3_object in parsed_s3_values { let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await; tracing::warn!("Enriching s3 arg value with etag: {:?}", etag); @@ -1169,7 +1182,10 @@ pub async fn get_cached_resource_value_if_valid( let object_store_resource_opt: Option = if s3_etags.is_empty() { None } else { - get_workspace_s3_resource_path(_db, &client, _workspace_id, _job_id).await + get_workspace_s3_resource_path(_db, &client, _workspace_id) + .await + .ok() + .flatten() }; if !s3_etags.is_empty() && object_store_resource_opt.is_none() { @@ -1205,8 +1221,7 @@ pub async fn save_in_cache( let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64; #[cfg(feature = "parquet")] - let s3_etags = - arg_value_hash_additions(db, _client, job.workspace_id.as_str(), &job.id, r).await; + let s3_etags = arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await; #[cfg(feature = "parquet")] let s3_etags = if s3_etags.is_empty() { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0ab78c1db5..57bc99f859 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -392,6 +392,18 @@ impl AuthedClient { .await?) } + pub async fn get_id_token(&self, audience: &str) -> anyhow::Result { + let url = format!( + "{}/api/w/{}/oidc/token/{}", + self.base_internal_url, self.workspace, audience + ); + let response = self.get(&url, vec![]).await?; + match response.status().as_u16() { + 200u16 => Ok(response.json::().await?), + _ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())), + } + } + pub async fn get_resource_value(&self, path: &str) -> anyhow::Result { let url = format!( "{}/api/w/{}/resources/get_value/{}", @@ -3413,11 +3425,7 @@ async fn handle_dependency_job('Tabs') + + $: isSelected = exact ? $selected == value : $selected?.startsWith(value) const fontSizeClasses = { xs: 'text-xs', @@ -22,22 +26,20 @@ lg: 'text-lg', xl: 'text-xl' } - - const { selected, update, hashNavigation } = getContext('Tabs') - -
- - {#if s3ResourceSettings.publicResource === true} - - S3 resource public access is ON, which means that the entire content of the S3 bucket - will be accessible to all the users of this workspace regardless of whether they have - access the resource or not. Similarly, certain Windmill SDK endpoints can be used in - scripts to access the resource details, including public and private keys. - - {/if} + +
+ {#if s3ResourceSettings.resourceType == 's3'} +
+ + {#if s3ResourceSettings.publicResource === true} + + S3 resource public access is ON, which means that the entire content of the S3 + bucket will be accessible to all the users of this workspace regardless of whether + they have access the resource or not. Similarly, certain Windmill SDK endpoints can + be used in scripts to access the resource details, including public and private + keys. + + {/if} +
+ {/if}