feat: implement s3 oidc support + azure workload identity support

This commit is contained in:
Ruben Fiszel
2024-03-08 22:51:03 +01:00
parent 41a1c78595
commit 4578ed32da
17 changed files with 694 additions and 188 deletions
+347 -17
View File
@@ -398,7 +398,7 @@ checksum = "beec3f8fb8f710b7be84ccd1716e17f38f2868168355cab5f2f168ae988e767e"
dependencies = [
"base64 0.21.7",
"bytes",
"http",
"http 0.2.12",
"rand 0.8.5",
"reqwest",
"serde",
@@ -575,6 +575,300 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "aws-config"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b96342ea8948ab9bef3e6234ea97fc32e2d8a88d8fb6a084e52267317f94b6b"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-sdk-sso",
"aws-sdk-ssooidc",
"aws-sdk-sts",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.0.1",
"hex",
"http 0.2.12",
"hyper",
"ring 0.17.8",
"time",
"tokio",
"tracing",
"zeroize",
]
[[package]]
name = "aws-credential-types"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "273fa47dafc9ef14c2c074ddddbea4561ff01b7f68d5091c0e9737ced605c01d"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
"aws-smithy-types",
"zeroize",
]
[[package]]
name = "aws-runtime"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e38bab716c8bf07da24be07ecc02e0f5656ce8f30a891322ecdcb202f943b85"
dependencies = [
"aws-credential-types",
"aws-sigv4",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand 2.0.1",
"http 0.2.12",
"http-body",
"percent-encoding",
"pin-project-lite",
"tracing",
"uuid 1.7.0",
]
[[package]]
name = "aws-sdk-sso"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d84bd3925a17c9adbf6ec65d52104a44a09629d8f70290542beeee69a95aee7f"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
"bytes",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-ssooidc"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c2dae39e997f58bc4d6292e6244b26ba630c01ab671b6f9f44309de3eb80ab8"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-types",
"bytes",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-sts"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17fd9a53869fee17cea77e352084e1aa71e2c5e323d974c13a9c2bcfd9544c7f"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-query",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"http 0.2.12",
"once_cell",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sigv4"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ada00a4645d7d89f296fe0ddbc3fe3554f03035937c849a05d37ddffc1f29a1"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
"form_urlencoded",
"hex",
"hmac",
"http 0.2.12",
"http 1.1.0",
"once_cell",
"percent-encoding",
"sha2 0.10.8",
"time",
"tracing",
]
[[package]]
name = "aws-smithy-async"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcf7f09a27286d84315dfb9346208abb3b0973a692454ae6d0bc8d803fcce3b4"
dependencies = [
"futures-util",
"pin-project-lite",
"tokio",
]
[[package]]
name = "aws-smithy-http"
version = "0.60.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6ca214a6a26f1b7ebd63aa8d4f5e2194095643023f9608edf99a58247b9d80d"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
"bytes-utils",
"futures-core",
"http 0.2.12",
"http-body",
"once_cell",
"percent-encoding",
"pin-project-lite",
"pin-utils",
"tracing",
]
[[package]]
name = "aws-smithy-json"
version = "0.60.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1af80ecf3057fb25fe38d1687e94c4601a7817c6a1e87c1b0635f7ecb644ace5"
dependencies = [
"aws-smithy-types",
]
[[package]]
name = "aws-smithy-query"
version = "0.60.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb27084f72ea5fc20033efe180618677ff4a2f474b53d84695cfe310a6526cbc"
dependencies = [
"aws-smithy-types",
"urlencoding",
]
[[package]]
name = "aws-smithy-runtime"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbb5fca54a532a36ff927fbd7407a7c8eb9c3b4faf72792ba2965ea2cad8ed55"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
"fastrand 2.0.1",
"h2",
"http 0.2.12",
"http-body",
"hyper",
"hyper-rustls",
"once_cell",
"pin-project-lite",
"pin-utils",
"rustls 0.21.10",
"tokio",
"tracing",
]
[[package]]
name = "aws-smithy-runtime-api"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22389cb6f7cac64f266fb9f137745a9349ced7b47e0d2ba503e9e40ede4f7060"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
"bytes",
"http 0.2.12",
"http 1.1.0",
"pin-project-lite",
"tokio",
"tracing",
"zeroize",
]
[[package]]
name = "aws-smithy-types"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f081da5481210523d44ffd83d9f0740320050054006c719eae0232d411f024d3"
dependencies = [
"base64-simd",
"bytes",
"bytes-utils",
"futures-core",
"http 0.2.12",
"http-body",
"itoa",
"num-integer",
"pin-project-lite",
"pin-utils",
"ryu",
"serde",
"time",
"tokio",
"tokio-util",
]
[[package]]
name = "aws-smithy-xml"
version = "0.60.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fccd8f595d0ca839f9f2548e66b99514a85f92feb4c01cf2868d93eb4888a42"
dependencies = [
"xmlparser",
]
[[package]]
name = "aws-types"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d07c63521aa1ea9a9f92a701f1a08ce3fd20b46c6efc0d5c8947c1fd879e3df1"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
"aws-smithy-runtime-api",
"aws-smithy-types",
"http 0.2.12",
"rustc_version 0.4.0",
"tracing",
]
[[package]]
name = "axum"
version = "0.6.20"
@@ -587,7 +881,7 @@ dependencies = [
"bytes",
"futures-util",
"headers",
"http",
"http 0.2.12",
"http-body",
"hyper",
"itoa",
@@ -617,7 +911,7 @@ dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http 0.2.12",
"http-body",
"mime",
"rustversion",
@@ -978,6 +1272,16 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
[[package]]
name = "bytes-utils"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35"
dependencies = [
"bytes",
"either",
]
[[package]]
name = "bzip2"
version = "0.4.4"
@@ -1759,7 +2063,7 @@ dependencies = [
"deno_core",
"deno_tls",
"dyn-clone",
"http",
"http 0.2.12",
"pin-project",
"reqwest",
"serde",
@@ -3002,7 +3306,7 @@ dependencies = [
"futures-core",
"futures-sink",
"futures-util",
"http",
"http 0.2.12",
"indexmap 2.2.5",
"slab",
"tokio",
@@ -3071,7 +3375,7 @@ dependencies = [
"base64 0.21.7",
"bytes",
"headers-core",
"http",
"http 0.2.12",
"httpdate",
"mime",
"sha1",
@@ -3083,7 +3387,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429"
dependencies = [
"http",
"http 0.2.12",
]
[[package]]
@@ -3175,6 +3479,17 @@ dependencies = [
"itoa",
]
[[package]]
name = "http"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.6"
@@ -3182,7 +3497,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
dependencies = [
"bytes",
"http",
"http 0.2.12",
"pin-project-lite",
]
@@ -3202,7 +3517,7 @@ dependencies = [
"async-channel 1.9.0",
"base64 0.13.1",
"futures-lite 1.13.0",
"http",
"http 0.2.12",
"infer",
"pin-project-lite",
"rand 0.7.3",
@@ -3242,7 +3557,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2",
"http",
"http 0.2.12",
"http-body",
"httparse",
"httpdate",
@@ -3262,8 +3577,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590"
dependencies = [
"futures-util",
"http",
"http 0.2.12",
"hyper",
"log",
"rustls 0.21.10",
"rustls-native-certs",
"tokio",
@@ -4251,7 +4567,7 @@ dependencies = [
"base64 0.13.1",
"chrono",
"getrandom 0.2.12",
"http",
"http 0.2.12",
"rand 0.8.5",
"reqwest",
"serde",
@@ -4356,7 +4672,7 @@ dependencies = [
"dyn-clone",
"ed25519-dalek",
"hmac",
"http",
"http 0.2.12",
"itertools 0.10.5",
"log",
"oauth2",
@@ -5375,7 +5691,7 @@ source = "git+https://github.com/oxidecomputer/progenitor?rev=3d96016ae8d422e905
dependencies = [
"getopts",
"heck",
"http",
"http 0.2.12",
"indexmap 1.9.3",
"openapiv3",
"proc-macro2",
@@ -5763,6 +6079,12 @@ dependencies = [
"regex-syntax 0.8.2",
]
[[package]]
name = "regex-lite"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e"
[[package]]
name = "regex-syntax"
version = "0.6.29"
@@ -5813,7 +6135,7 @@ dependencies = [
"futures-core",
"futures-util",
"h2",
"http",
"http 0.2.12",
"http-body",
"hyper",
"hyper-rustls",
@@ -8241,7 +8563,7 @@ dependencies = [
"axum-core",
"cookie",
"futures-util",
"http",
"http 0.2.12",
"parking_lot",
"pin-project-lite",
"tower-layer",
@@ -8258,7 +8580,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-util",
"http",
"http 0.2.12",
"http-body",
"http-range-header",
"pin-project-lite",
@@ -9199,6 +9521,8 @@ name = "windmill-common"
version = "1.287.1"
dependencies = [
"anyhow",
"aws-config",
"aws-sdk-sts",
"axum",
"chrono",
"cron",
@@ -9646,6 +9970,12 @@ dependencies = [
"rustix 0.38.31",
]
[[package]]
name = "xmlparser"
version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xxhash-rust"
version = "0.8.10"
+2
View File
@@ -227,3 +227,5 @@ polars-io = { version = "0.35.4", features = ["csv"] }
object_store = { version = "0.8.0", features = ["aws", "azure"] }
openidconnect = { version = "3.4.0" }
aws-config = "^1"
aws-sdk-sts = "^1"
+1 -1
View File
@@ -1 +1 @@
7733647d5d3026654d3dcc279ec0cae934532f8a
e419b8dad3a83783b4faaacd2396245c0d776f92
+1
View File
@@ -87,3 +87,4 @@ polars-io = { workspace = true, optional = true}
object_store = { workspace = true, optional = true}
openidconnect = { workspace = true, optional = true}
pin-project.workspace = true
+1 -1
View File
@@ -9673,7 +9673,7 @@ components:
properties:
type:
type: string
enum: ["S3Storage", "AzureBlobStorage"]
enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"]
s3_resource_path:
type: string
azure_blob_resource_path:
+3 -1
View File
@@ -10,7 +10,7 @@ enterprise = []
prometheus = ["dep:prometheus"]
flamegraph = ["dep:tracing-flame"]
loki = ["dep:tracing-loki"]
parquet = ["dep:object_store"]
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
[lib]
name = "windmill_common"
@@ -44,3 +44,5 @@ tracing-loki = { version = "^0", optional = true }
magic-crypt.workspace = true
object_store = { workspace = true, optional = true }
prometheus = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true }
aws-sdk-sts = { workspace = true, optional = true }
+1 -1
View File
@@ -90,7 +90,7 @@ impl IntoResponse for Error {
if matches!(status, axum::http::StatusCode::NOT_FOUND) {
tracing::warn!(message = e.to_string());
} else {
tracing::error!(message = e.to_string());
tracing::error!(message = e.to_string(), error = ?e);
};
axum::response::Response::builder()
+1
View File
@@ -0,0 +1 @@
/git/windmill/../windmill-ee-private/windmill-common/src/job_s3_helpers_ee.rs
+3
View File
@@ -24,10 +24,13 @@ pub mod flow_status;
pub mod flows;
pub mod global_settings;
pub mod job_metrics;
#[cfg(feature = "parquet")]
pub mod job_s3_helpers_ee;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
pub mod s3_helpers;
pub mod schedule;
pub mod scripts;
pub mod server;
+84 -11
View File
@@ -15,6 +15,8 @@ use std::sync::Arc;
pub enum LargeFileStorage {
S3Storage(S3Storage),
AzureBlobStorage(AzureBlobStorage),
S3AwsOidc(S3Storage),
AzureWorkloadIdentity(AzureBlobStorage),
// TODO: Add a filesystem type here in the future if needed
}
@@ -38,6 +40,14 @@ pub enum ObjectStoreResource {
AzureBlobResource(AzureBlobResource),
}
#[derive(Deserialize, Debug)]
pub enum StorageResourceType {
S3,
AzureBlob,
S3AwsOidc,
AzureWorkloadIdentity,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct S3Resource {
#[serde(rename = "bucket")]
@@ -52,7 +62,8 @@ pub struct S3Resource {
#[serde(rename = "secretKey")]
pub secret_key: Option<String>,
#[serde(rename = "pathStyle")]
pub path_style: bool,
pub path_style: Option<bool>,
pub token: Option<String>,
pub port: Option<u16>,
}
@@ -61,13 +72,29 @@ pub struct AzureBlobResource {
#[serde(rename = "endpoint")]
pub endpoint: Option<String>,
#[serde(rename = "useSSL")]
pub use_ssl: bool,
pub use_ssl: Option<bool>,
#[serde(rename = "accountName")]
pub account_name: String,
#[serde(rename = "tenantId")]
pub tenant_id: Option<String>,
#[serde(rename = "clientId")]
pub client_id: Option<String>,
#[serde(rename = "containerName")]
pub container_name: String,
#[serde(rename = "accessKey")]
pub access_key: Option<String>,
#[serde(rename = "federatedTokenFile")]
pub federated_token_file: Option<String>,
}
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct S3AwsOidcResource {
#[serde(rename = "bucket")]
pub bucket: String,
pub region: Option<String>,
#[serde(rename = "roleArn")]
pub role_arn: String,
pub audience: Option<String>,
}
#[derive(Deserialize, Clone)]
@@ -96,14 +123,27 @@ pub async fn get_etag_or_empty(
.flatten();
}
pub fn render_endpoint(raw_endpoint: String, use_ssl: bool, port: Option<u16>) -> String {
pub fn render_endpoint(
raw_endpoint: String,
use_ssl: bool,
port: Option<u16>,
path_style: Option<bool>,
bucket: String,
) -> String {
let url_with_prefix =
if raw_endpoint.starts_with("http://") || raw_endpoint.starts_with("https://") {
raw_endpoint.clone()
} else if use_ssl {
format!("https://{}", raw_endpoint)
} else {
format!("http://{}", raw_endpoint)
let scheme = if use_ssl { "https" } else { "http" };
format!(
"{}://{}",
scheme,
if path_style.unwrap_or(true) {
raw_endpoint
} else {
format!("{}.{}", bucket, raw_endpoint)
}
)
};
if port.is_some() {
format!("{}:{}", url_with_prefix, port.unwrap())
@@ -127,7 +167,13 @@ pub fn build_object_store_client(
#[cfg(feature = "parquet")]
fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result<Arc<dyn ObjectStore>> {
let s3_resource = s3_resource_ref.clone();
let endpoint = render_endpoint(s3_resource.endpoint, s3_resource.use_ssl, s3_resource.port);
let endpoint = render_endpoint(
s3_resource.endpoint,
s3_resource.use_ssl,
s3_resource.port,
s3_resource.path_style,
s3_resource.bucket.clone(),
);
let mut store_builder = AmazonS3Builder::new()
.with_client_options(ClientOptions::new().with_timeout_disabled()) // TODO: make it configurable maybe
@@ -144,13 +190,19 @@ fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result<Arc<dyn Object
store_builder = store_builder.with_access_key_id(key);
}
}
if let Some(token) = s3_resource.token {
if token != "" {
store_builder = store_builder.with_token(token);
}
}
if let Some(secret_key) = s3_resource.secret_key {
if secret_key != "" {
store_builder = store_builder.with_secret_access_key(secret_key);
}
}
if !s3_resource.path_style {
store_builder = store_builder.with_virtual_hosted_style_request(s3_resource.path_style);
if !s3_resource.path_style.unwrap_or(true) {
store_builder = store_builder.with_virtual_hosted_style_request(true);
}
let store = store_builder.build().map_err(|err| {
@@ -175,14 +227,35 @@ fn build_azure_blob_client(
.with_account(blob_resource.account_name)
.with_container_name(blob_resource.container_name);
if let Some(federated_token_file) = blob_resource.federated_token_file {
if federated_token_file != "" {
store_builder = store_builder.with_federated_token_file(federated_token_file);
}
}
if let Some(tenant_id) = blob_resource.tenant_id {
if tenant_id != "" {
store_builder = store_builder.with_tenant_id(tenant_id);
}
}
if let Some(client_id) = blob_resource.client_id {
if client_id != "" {
store_builder = store_builder.with_client_id(client_id);
}
}
if let Some(endpoint) = blob_resource.endpoint {
if endpoint != "" {
let endpoint = render_endpoint(endpoint, blob_resource.use_ssl, None);
let endpoint = render_endpoint(
endpoint,
blob_resource.use_ssl.unwrap_or(false),
None,
None,
"".to_string(),
);
store_builder = store_builder.with_endpoint(endpoint)
}
}
if !blob_resource.use_ssl {
if !blob_resource.use_ssl.unwrap_or(false) {
store_builder = store_builder.with_allow_http(true)
}
+46 -31
View File
@@ -19,8 +19,7 @@ use windmill_common::error::to_anyhow;
use windmill_common::jobs::ENTRYPOINT_OVERRIDE;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::{
get_etag_or_empty, AzureBlobResource, LargeFileStorage, ObjectStoreResource, S3Object,
S3Resource,
get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object,
};
use windmill_common::worker::{CLOUD_HOSTED, WORKER_CONFIG};
use windmill_common::{
@@ -1028,8 +1027,7 @@ pub async fn hash_args(
let arg_value = hm.get(k).unwrap();
#[cfg(feature = "parquet")]
let arg_additions =
arg_value_hash_additions(_db, _client, _workspace_id, _job_id, hm.get(k).unwrap())
.await;
arg_value_hash_additions(_db, _client, _workspace_id, hm.get(k).unwrap()).await;
arg_value.get().hash(&mut dh);
#[cfg(feature = "parquet")]
for (_, arg_addition) in arg_additions {
@@ -1047,8 +1045,11 @@ async fn get_workspace_s3_resource_path(
db: &DB,
client: &AuthedClient,
workspace_id: &str,
job_id: &Uuid,
) -> Option<ObjectStoreResource> {
) -> windmill_common::error::Result<Option<ObjectStoreResource>> {
use windmill_common::{
job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType,
};
let raw_lfs_opt = sqlx::query_scalar!(
"SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1",
workspace_id
@@ -1060,33 +1061,46 @@ async fn get_workspace_s3_resource_path(
.map(|val| serde_json::from_value::<LargeFileStorage>(val).ok())
.flatten();
match raw_lfs_opt {
let (rt, path) = match raw_lfs_opt {
Some(LargeFileStorage::S3Storage(s3_storage)) => {
let resource_path = s3_storage.s3_resource_path.trim_start_matches("$res:");
let s3_resource = client
.get_resource_value_interpolated::<S3Resource>(
&resource_path,
Some(job_id.to_string()),
)
.await
.ok();
s3_resource.map(|resource| ObjectStoreResource::S3Resource(resource))
(StorageResourceType::S3, resource_path.to_string())
}
Some(LargeFileStorage::AzureBlobStorage(azure_blob_storage)) => {
let resource_path = azure_blob_storage
.azure_blob_resource_path
.trim_start_matches("$res:");
let azure_blob_resource = client
.get_resource_value_interpolated::<AzureBlobResource>(
&resource_path,
Some(job_id.to_string()),
)
.await
.ok();
azure_blob_resource.map(|resource| ObjectStoreResource::AzureBlobResource(resource))
(StorageResourceType::AzureBlob, resource_path.to_string())
}
None => None,
}
Some(LargeFileStorage::S3AwsOidc(s3_aws_oidc)) => {
let resource_path = s3_aws_oidc.s3_resource_path.trim_start_matches("$res:");
(StorageResourceType::S3AwsOidc, resource_path.to_string())
}
Some(LargeFileStorage::AzureWorkloadIdentity(azure)) => {
let resource_path = azure.azure_blob_resource_path.trim_start_matches("$res:");
(
StorageResourceType::AzureWorkloadIdentity,
resource_path.to_string(),
)
}
None => {
return Ok(None);
}
};
let client2 = client.clone();
let token_fn = |audience: String| async move {
client2
.get_id_token(&audience)
.await
.map_err(|e| windmill_common::error::Error::from(e))
};
let s3_resource_value_raw = client
.get_resource_value::<serde_json::Value>(path.as_str())
.await?;
get_s3_resource_internal(rt, s3_resource_value_raw, token_fn)
.await
.map(Some)
}
#[cfg(feature = "parquet")]
@@ -1094,7 +1108,6 @@ async fn arg_value_hash_additions(
db: &DB,
client: &AuthedClient,
workspace_id: &str,
job_id: &Uuid,
raw_value: &Box<RawValue>,
) -> HashMap<String, String> {
let mut result: HashMap<String, String> = HashMap::new();
@@ -1105,8 +1118,8 @@ async fn arg_value_hash_additions(
return result;
}
let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id, job_id).await;
if let Some(s3_resource) = s3_resource_opt {
let s3_resource_opt = get_workspace_s3_resource_path(db, client, workspace_id).await;
if let Some(s3_resource) = s3_resource_opt.ok().flatten() {
for s3_object in parsed_s3_values {
let etag = get_etag_or_empty(&s3_resource, s3_object.clone()).await;
tracing::warn!("Enriching s3 arg value with etag: {:?}", etag);
@@ -1169,7 +1182,10 @@ pub async fn get_cached_resource_value_if_valid(
let object_store_resource_opt: Option<ObjectStoreResource> = if s3_etags.is_empty() {
None
} else {
get_workspace_s3_resource_path(_db, &client, _workspace_id, _job_id).await
get_workspace_s3_resource_path(_db, &client, _workspace_id)
.await
.ok()
.flatten()
};
if !s3_etags.is_empty() && object_store_resource_opt.is_none() {
@@ -1205,8 +1221,7 @@ pub async fn save_in_cache(
let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64;
#[cfg(feature = "parquet")]
let s3_etags =
arg_value_hash_additions(db, _client, job.workspace_id.as_str(), &job.id, r).await;
let s3_etags = arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await;
#[cfg(feature = "parquet")]
let s3_etags = if s3_etags.is_empty() {
+13 -5
View File
@@ -392,6 +392,18 @@ impl AuthedClient {
.await?)
}
pub async fn get_id_token(&self, audience: &str) -> anyhow::Result<String> {
let url = format!(
"{}/api/w/{}/oidc/token/{}",
self.base_internal_url, self.workspace, audience
);
let response = self.get(&url, vec![]).await?;
match response.status().as_u16() {
200u16 => Ok(response.json::<String>().await?),
_ => Err(anyhow::anyhow!(response.text().await.unwrap_or_default())),
}
}
pub async fn get_resource_value<T: DeserializeOwned>(&self, path: &str) -> anyhow::Result<T> {
let url = format!(
"{}/api/w/{}/resources/get_value/{}",
@@ -3413,11 +3425,7 @@ async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync + Clo
&job.created_by,
&db,
&w_id,
DeployedObject::Script {
hash,
path: script_path.to_string(),
parent_path: parent_path.clone(),
},
DeployedObject::Script { hash, path: script_path.to_string(), parent_path: None },
deployment_message.clone(),
rsmq.clone(),
false,
+5 -5
View File
@@ -202,8 +202,10 @@ async function updateScriptLock(
}
);
let responseText = "reading response failed";
try {
const response = await rawResponse.json();
responseText = await rawResponse.text();
const response = JSON.parse(responseText);
const lock = response.lock;
if (lock === undefined) {
throw new Error(
@@ -213,11 +215,9 @@ async function updateScriptLock(
);
}
metadataContent.lock = lock;
} catch {
} catch (e) {
throw new Error(
`Failed to generate lockfile. Status was: ${
rawResponse.statusText
}, ${await rawResponse.text()}`
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
);
}
}
@@ -61,16 +61,6 @@
jobId: undefined
})
if (rowContext && rowInputs) {
const inputOutput = { result: outputs.result.peak(), loading: false }
rowInputs.set(id, inputOutput)
}
if (iterContext && listInputs) {
const inputOutput = { result: outputs.result.peak(), loading: false }
listInputs.set(id, inputOutput)
}
if (controls) {
$componentControl[id] = controls
}
@@ -12,8 +12,12 @@
export let selectedStyle = ''
export let id: string | undefined = undefined
export let active: boolean | undefined = false
export let exact = false
export let disabled: boolean = false
const { selected, update, hashNavigation } = getContext<TabsContext>('Tabs')
$: isSelected = exact ? $selected == value : $selected?.startsWith(value)
const fontSizeClasses = {
xs: 'text-xs',
@@ -22,22 +26,20 @@
lg: 'text-lg',
xl: 'text-xl'
}
const { selected, update, hashNavigation } = getContext<TabsContext>('Tabs')
</script>
<button
class={twMerge(
'border-b-2 py-1 px-2 cursor-pointer transition-all z-10 ease-linear font-normal text-primary',
$selected?.startsWith(value)
isSelected
? 'wm-tab-active font-main'
: 'border-gray-300 dark:border-gray-600 border-opacity-0 hover:border-opacity-100 ',
fontSizeClasses[size],
c,
$selected?.startsWith(value) ? selectedClass : '',
isSelected ? selectedClass : '',
disabled ? 'cursor-not-allowed text-tertiary' : ''
)}
style={`${style} ${$selected?.startsWith(value) ? selectedStyle : ''}`}
style={`${style} ${isSelected ? selectedStyle : ''}`}
on:click={() => {
if (hashNavigation) {
window.location.hash = value
@@ -380,7 +380,7 @@
<DrawerContent title={resourceTypeViewerObj.rt} on:close={resourceTypeViewer.closeDrawer}>
<div>
<h1 class="mb-8 mt-4"><IconedResourceType name={resourceTypeViewerObj.rt} /></h1>
<div class="py-2 box prose mb-8">
<div class="py-2 box prose mb-8 text-secondary">
{resourceTypeViewerObj.description ?? ''}
</div>
<SchemaViewer schema={resourceTypeViewerObj.schema} />
@@ -547,8 +547,8 @@
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/resources_and_types"
>
Every resource has a Resource Type attached to it which contains its schema and make
it easy in scripts and flows to accept only resources of a specific resource type.
Every resource has a Resource Type attached to it which contains its schema and make it
easy in scripts and flows to accept only resources of a specific resource type.
</Tooltip>
</div>
</Tab>
@@ -92,7 +92,7 @@
let errorHandlerMutedOnCancel: boolean | undefined = undefined
let openaiResourceInitialPath: string | undefined = undefined
let s3ResourceSettings: {
resourceType: 's3' | 'azure_blob'
resourceType: 's3' | 'azure_blob' | 's3_aws_oidc' | 'azure_workload_identity'
resourcePath: string | undefined
publicResource: boolean | undefined
}
@@ -226,6 +226,12 @@
if (s3ResourceSettings.resourceType === 'azure_blob') {
params['type'] = LargeFileStorage.type.AZURE_BLOB_STORAGE
params['azure_blob_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 'azure_workload_identity') {
params['type'] = LargeFileStorage.type.AZURE_WORKLOAD_IDENTITY
params['azure_blob_resource_path'] = resourcePathWithPrefix
} else if (s3ResourceSettings.resourceType === 's3_aws_oidc') {
params['type'] = LargeFileStorage.type.S3AWS_OIDC
params['s3_resource_path'] = resourcePathWithPrefix
} else {
params['type'] = LargeFileStorage.type.S3STORAGE
params['s3_resource_path'] = resourcePathWithPrefix
@@ -427,6 +433,20 @@
resourcePath: settings.large_file_storage?.azure_blob_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else if (
settings.large_file_storage?.type === LargeFileStorage.type.AZURE_WORKLOAD_IDENTITY
) {
s3ResourceSettings = {
resourceType: 'azure_workload_identity',
resourcePath: settings.large_file_storage?.azure_blob_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else if (settings.large_file_storage?.type === LargeFileStorage.type.S3AWS_OIDC) {
s3ResourceSettings = {
resourceType: 's3_aws_oidc',
resourcePath: settings.large_file_storage?.s3_resource_path?.replace('$res:', ''),
publicResource: settings.large_file_storage?.public_resource
}
} else {
s3ResourceSettings = {
resourceType: 's3',
@@ -658,14 +678,21 @@
<WorkspaceUserSettings />
{:else if tab == 'deploy_to'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Link this workspace to another Staging / Prod workspace </div>
<div class="text-tertiary text-xs">
Connecting this workspace with another staging/production workspace enables web-based deployment to that workspace.
<a href="https://www.windmill.dev/docs/core_concepts/staging_prod" target="_blank" class="text-blue-500">Learn more</a>.
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">
Link this workspace to another Staging / Prod workspace
</div>
<div class="text-tertiary text-xs">
Connecting this workspace with another staging/production workspace enables web-based
deployment to that workspace.
<a
href="https://www.windmill.dev/docs/core_concepts/staging_prod"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
</div>
</div>
{#if $enterpriseLicense}
<DeployToSetting bind:workspaceToDeployTo />
{:else}
@@ -684,7 +711,11 @@
<div class="text-tertiary text-xs">
Connect your Windmill workspace to your Slack workspace to trigger a script or a flow
with a '/windmill' command or to configure Slack error handlers.
<a href="https://www.windmill.dev/docs/integrations/slack" target="_blank" class="text-blue-500">Learn more</a>.
<a
href="https://www.windmill.dev/docs/integrations/slack"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
@@ -772,9 +803,9 @@
</div>
{:else if tab == 'export_delete'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">Export or Delete workspace</div>
</div>
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">Export or Delete workspace</div>
</div>
</div>
<PageHeader title="Export workspace" primary={false} />
<div class="flex justify-start">
@@ -835,19 +866,25 @@
</div>
{:else if tab == 'webhook'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Workspace Webhook</div>
<div class="text-tertiary text-xs">
Connect your Windmill workspace to an external service to sync or get notified about any change.
<a href="https://www.windmill.dev/docs/core_concepts/webhooks#workspace-webhook" target="_blank" class="text-blue-500">Learn more</a>.
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Workspace Webhook</div>
<div class="text-tertiary text-xs">
Connect your Windmill workspace to an external service to sync or get notified about any
change.
<a
href="https://www.windmill.dev/docs/core_concepts/webhooks#workspace-webhook"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
</div>
</div>
<div class="flex flex-col gap-4 my-4">
<div class="flex flex-col gap-1">
<div class=" text-primary text-base font-semibold"> URL to send requests to</div>
<div class="text-tertiary text-xs">
This URL will be POSTed to with a JSON body depending on the type of event. The type is indicated by the type field. The other fields are dependent on the type.
This URL will be POSTed to with a JSON body depending on the type of event. The type is
indicated by the type field. The other fields are dependent on the type.
</div>
</div>
</div>
@@ -868,13 +905,19 @@
<div class=" text-primary text-lg font-semibold"> Workspace Error Handler</div>
<div class="text-tertiary text-xs">
Define a script or flow to be executed automatically in case of error in the workspace.
<a href="https://www.windmill.dev/docs/core_concepts/error_handling#workspace-error-handler" target="_blank" class="text-blue-500">Learn more</a>.
<a
href="https://www.windmill.dev/docs/core_concepts/error_handling#workspace-error-handler"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
</div>
</div>
<div class="flex flex-col gap-4 my-4">
<div class="flex flex-col gap-1">
<div class=" text-primary text-base font-semibold"> Script or flow to run as error handler</div>
<div class=" text-primary text-base font-semibold">
Script or flow to run as error handler</div
>
</div>
</div>
<ErrorOrRecoveryHandler
@@ -934,16 +977,21 @@
</div>
{:else if tab == 'openai'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Windmill AI</div>
<div class="text-tertiary text-xs">
Select an OpenAI resource to unlock Windmill AI features.
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Windmill AI</div>
<div class="text-tertiary text-xs">
Select an OpenAI resource to unlock Windmill AI features.
</div>
<div class="text-tertiary text-xs">
Windmill AI uses OpenAI's GPT-3.5-turbo for code completion and GPT-4 Turbo for all
other AI features.
<a
href="https://www.windmill.dev/docs/core_concepts/ai_generation"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
<div class="text-tertiary text-xs">
Windmill AI uses OpenAI's GPT-3.5-turbo for code completion and GPT-4 Turbo for all other AI features.
<a href="https://www.windmill.dev/docs/core_concepts/ai_generation" target="_blank" class="text-blue-500">Learn more</a>.
</div>
</div>
</div>
<div class="mt-5 flex gap-1">
{#key [openaiResourceInitialPath, usingOpenaiClientCredentialsOauth]}
@@ -971,14 +1019,18 @@
</div>
{:else if tab == 'windmill_lfs'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">S3 Storage</div>
<div class="text-tertiary text-xs">
Connect your Windmill workspace to your S3 bucket or your Azure Blob storage.
<a href="https://www.windmill.dev/docs/core_concepts/persistent_storage#connect-your-windmill-workspace-to-your-s3-bucket-or-your-azure-blob-storage" target="_blank" class="text-blue-500">Learn more</a>.
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">S3 Storage</div>
<div class="text-tertiary text-xs">
Connect your Windmill workspace to your S3 bucket or your Azure Blob storage.
<a
href="https://www.windmill.dev/docs/core_concepts/persistent_storage#connect-your-windmill-workspace-to-your-s3-bucket-or-your-azure-blob-storage"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
</div>
</div>
{#if !$enterpriseLicense}
<Alert type="info" title="S3 storage is limited to 20 files in Windmill CE">
Windmill S3 bucket browser will not work for buckets containing more than 20 files and
@@ -987,45 +1039,55 @@
</Alert>
{/if}
{#if s3ResourceSettings}
<div class="mt-5 flex gap-1">
{#key s3ResourceSettings.resourcePath}
<div class="mt-5">
<div class="w-full">
<Tabs bind:selected={s3ResourceSettings.resourceType}>
<Tab exact size="xs" value="s3">S3</Tab>
<Tab size="xs" value="azure_blob">Azure Blob</Tab>
<Tab exact size="xs" value="s3_aws_oidc">AWS OIDC</Tab>
<Tab size="xs" value="azure_workload_identity">Azure Workload Identity</Tab>
</Tabs>
</div>
<div class="w-full flex gap-1 mt-4">
<ResourcePicker
resourceType="s3,azure_blob"
resourceType={s3ResourceSettings.resourceType}
bind:value={s3ResourceSettings.resourcePath}
bind:valueType={s3ResourceSettings.resourceType}
/>
{/key}
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(s3ResourceSettings.resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.(undefined)
}
}}>Browse content (save first)</Button
>
</div>
<div class="flex flex-col mt-5 mb-1 gap-1">
<Toggle
disabled={emptyString(s3ResourceSettings.resourcePath)}
bind:checked={s3ResourceSettings.publicResource}
options={{
right: 'S3 resource details can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the S3 bucket, as well as the resource details. this effectively by-pass the permissions set on the resource and makes it public to everyone.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
<Alert type="warning" title="S3 bucket content and resource details are shared">
S3 resource public access is ON, which means that the entire content of the S3 bucket
will be accessible to all the users of this workspace regardless of whether they have
access the resource or not. Similarly, certain Windmill SDK endpoints can be used in
scripts to access the resource details, including public and private keys.
</Alert>
{/if}
<Button
size="sm"
variant="contained"
color="dark"
disabled={emptyString(s3ResourceSettings.resourcePath)}
on:click={async () => {
if ($workspaceStore) {
s3FileViewer?.open?.(undefined)
}
}}>Browse content (save first)</Button
>
</div>
</div>
{#if s3ResourceSettings.resourceType == 's3'}
<div class="flex flex-col mt-5 mb-1 gap-1">
<Toggle
disabled={emptyString(s3ResourceSettings.resourcePath)}
bind:checked={s3ResourceSettings.publicResource}
options={{
right: 'S3 resource details can be accessed by all users of this workspace',
rightTooltip:
'If set, all users of this workspace will have access the to entire content of the S3 bucket, as well as the resource details. this effectively by-pass the permissions set on the resource and makes it public to everyone.'
}}
/>
{#if s3ResourceSettings.publicResource === true}
<Alert type="warning" title="S3 bucket content and resource details are shared">
S3 resource public access is ON, which means that the entire content of the S3
bucket will be accessible to all the users of this workspace regardless of whether
they have access the resource or not. Similarly, certain Windmill SDK endpoints can
be used in scripts to access the resource details, including public and private
keys.
</Alert>
{/if}
</div>
{/if}
<div class="flex mt-5 mb-5 gap-1">
<Button
color="blue"
@@ -1038,16 +1100,20 @@
</div>
{/if}
{:else if tab == 'git_sync'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Git Sync </div>
<div class="text-tertiary text-xs">
Connect the Windmill workspace to a Git repository to automatically commit and push
scripts, flows, and apps to the repository on each deploy.
<a href="https://www.windmill.dev/docs/advanced/git_sync" target="_blank" class="text-blue-500">Learn more</a>.
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold"> Git Sync </div>
<div class="text-tertiary text-xs">
Connect the Windmill workspace to a Git repository to automatically commit and push
scripts, flows, and apps to the repository on each deploy.
<a
href="https://www.windmill.dev/docs/advanced/git_sync"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
</div>
</div>
{#if !$enterpriseLicense}
<div class="mb-2" />
@@ -1425,16 +1491,22 @@ git push</code
{/if}
{:else if tab == 'default_app'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">Workspace default app</div>
<div class="text-tertiary text-xs">
If configured, users who are operators in this workspace will be redirected to this app automatically when logging into this workspace.
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">Workspace default app</div>
<div class="text-tertiary text-xs">
If configured, users who are operators in this workspace will be redirected to this app
automatically when logging into this workspace.
</div>
<div class="text-tertiary text-xs">
Make sure the default app is shared with all the operators of this workspace before
turning this feature on.
<a
href="https://www.windmill.dev/docs/apps/default_app"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
<div class="text-tertiary text-xs">
Make sure the default app is shared with all the operators of this workspace before turning this feature on.
<a href="https://www.windmill.dev/docs/apps/default_app" target="_blank" class="text-blue-500">Learn more</a>.
</div>
</div>
</div>
{#if !$enterpriseLicense}
<Alert type="info" title="Windmill EE only feature">
@@ -1458,16 +1530,23 @@ git push</code
</div>
{:else if tab == 'encryption'}
<div class="flex flex-col gap-4 my-8">
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">Workspace secret encryption</div>
<div class="text-tertiary text-xs">
When updating the encryption key of a workspace, all secrets will be re-encrypted with the new key and the previous key will be replaced by the new one.
<div class="flex flex-col gap-1">
<div class=" text-primary text-lg font-semibold">Workspace secret encryption</div>
<div class="text-tertiary text-xs">
When updating the encryption key of a workspace, all secrets will be re-encrypted with
the new key and the previous key will be replaced by the new one.
</div>
<div class="text-tertiary text-xs">
If you're manually updating the key to match another workspace key from another Windmill
instance, make sure not to use the 'SECRET_SALT' environment variable or, if you're
using it, make sure it the salt matches across both instances.
<a
href="https://www.windmill.dev/docs/core_concepts/workspace_secret_encryption"
target="_blank"
class="text-blue-500">Learn more</a
>.
</div>
</div>
<div class="text-tertiary text-xs">
If you're manually updating the key to match another workspace key from another Windmill instance, make sure not to use the 'SECRET_SALT' environment variable or, if you're using it, make sure it the salt matches across both instances.
<a href="https://www.windmill.dev/docs/core_concepts/workspace_secret_encryption" target="_blank" class="text-blue-500">Learn more</a>.
</div>
</div>
</div>
<div class="mt-5 flex gap-1 mb-10">
<Button