diff --git a/backend/Cargo.lock b/backend/Cargo.lock index fdf9c84227..1e08099f33 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -9416,6 +9416,7 @@ dependencies = [ "hyper", "itertools 0.12.0", "lazy_static", + "magic-crypt", "prometheus", "rand 0.8.5", "regex", diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 710747f841..b50a32d6e8 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -10,7 +10,6 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, users::{require_owner_of_path, OptAuthed}, - variables::build_crypt, webhook_util::{WebhookMessage, WebhookShared}, HTTP_CLIENT, }; @@ -39,6 +38,7 @@ use windmill_common::{ utils::{ http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath, }, + variables::build_crypt, }; use windmill_queue::{push, PushArgs, PushIsolationLevel, QueueTransaction}; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index c1e8f176fa..602afbb8ea 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -10,6 +10,7 @@ use axum::http::HeaderValue; use serde_json::value::RawValue; use std::collections::HashMap; use windmill_common::flow_status::RestartedFrom; +use windmill_common::variables::get_workspace_key; use crate::db::ApiAuthed; @@ -17,7 +18,6 @@ use crate::{ db::DB, users::{check_scopes, require_owner_of_path, OptAuthed}, utils::require_super_admin, - variables::get_workspace_key, }; use anyhow::Context; use axum::{ diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index 039f7c4add..b574a2e150 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -38,16 +38,13 @@ use windmill_common::jobs::JobPayload; use windmill_common::more_serde::maybe_number_opt; use windmill_common::users::username_to_permissioned_as; use windmill_common::utils::{not_found_if_none, now_from_db}; +use windmill_common::variables::build_crypt; use crate::db::ApiAuthed; use crate::saml::SamlSsoLogin; use crate::users::{login_externally, LoginUserInfo}; use crate::webhook_util::{InstanceEvent, WebhookShared}; -use crate::{ - db::DB, - variables::{build_crypt, encrypt}, - workspaces::WorkspaceSettings, -}; +use crate::{db::DB, variables::encrypt, workspaces::WorkspaceSettings}; use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRET}; use windmill_common::error::{self, to_anyhow, Error}; use windmill_common::oauth2::*; diff --git a/backend/windmill-api/src/openai.rs b/backend/windmill-api/src/openai.rs index 626f0f6330..3282c9cbd0 100644 --- a/backend/windmill-api/src/openai.rs +++ b/backend/windmill-api/src/openai.rs @@ -2,7 +2,6 @@ use std::collections::HashMap; use crate::{ db::{ApiAuthed, DB}, - variables::build_crypt, HTTP_CLIENT, }; @@ -17,7 +16,10 @@ use magic_crypt::MagicCryptTrait; use quick_cache::sync::Cache; use serde_json::value::RawValue; use windmill_audit::{audit_log, ActionKind}; -use windmill_common::error::{to_anyhow, Error}; +use windmill_common::{ + error::{to_anyhow, Error}, + variables::build_crypt, +}; use serde::Deserialize; diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index c810c0152b..758ba63677 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -25,7 +25,9 @@ use windmill_common::{ db::UserDB, error::{Error, JsonResult, Result}, utils::{not_found_if_none, StripPath}, - variables::{get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable}, + variables::{ + build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable, + }, }; use lazy_static::lazy_static; @@ -192,57 +194,6 @@ async fn get_value( .map(Json); } -pub async fn get_value_internal<'c>( - mut tx: Transaction<'c, Postgres>, - db: &DB, - w_id: &str, - path: &str, - username: &str, -) -> Result { - let variable_o = sqlx::query!( - "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable - LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id - ) - .fetch_optional(&mut *tx) - .await?; - - let variable = if let Some(variable) = variable_o { - variable - } else { - explain_variable_perm_error(path, w_id, db).await?; - unreachable!() - }; - - let r = if variable.is_secret { - audit_log( - &mut *tx, - username, - "variables.decrypt_secret", - ActionKind::Execute, - &w_id, - Some(&variable.path), - None, - ) - .await?; - let value = variable.value; - if variable.is_expired.unwrap_or(false) && variable.account.is_some() { - _refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await? - } else if !value.is_empty() { - let mc = build_crypt(&mut tx, &w_id).await?; - tx.commit().await?; - - mc.decrypt_base64_to_string(value) - .map_err(|e| Error::InternalErr(e.to_string()))? - } else { - "".to_string() - } - } else { - variable.value - }; - - Ok(r) -} - async fn explain_variable_perm_error( path: &str, w_id: &str, @@ -604,31 +555,55 @@ fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value { } } -pub async fn build_crypt<'c>( - db: &mut Transaction<'c, Postgres>, +pub async fn get_value_internal<'c>( + mut tx: Transaction<'c, Postgres>, + db: &DB, w_id: &str, -) -> Result { - let key = get_workspace_key(w_id, db).await?; - let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", key, salt) - } else { - key - }; - Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) -} - -pub async fn get_workspace_key<'c>( - w_id: &str, - db: &mut Transaction<'c, Postgres>, + path: &str, + username: &str, ) -> Result { - let key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - w_id + let variable_o = sqlx::query!( + "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable + LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id ) - .fetch_one(&mut **db) - .await - .map_err(|e| Error::InternalErr(format!("fetching workspace key: {e}")))?; - Ok(key) + .fetch_optional(&mut *tx) + .await?; + + let variable = if let Some(variable) = variable_o { + variable + } else { + explain_variable_perm_error(path, w_id, db).await?; + unreachable!() + }; + + let r = if variable.is_secret { + audit_log( + &mut *tx, + username, + "variables.decrypt_secret", + ActionKind::Execute, + &w_id, + Some(&variable.path), + None, + ) + .await?; + let value = variable.value; + if variable.is_expired.unwrap_or(false) && variable.account.is_some() { + _refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await? + } else if !value.is_empty() { + let mc = build_crypt(&mut tx, &w_id).await?; + tx.commit().await?; + + mc.decrypt_base64_to_string(value) + .map_err(|e| Error::InternalErr(e.to_string()))? + } else { + "".to_string() + } + } else { + variable.value + }; + + Ok(r) } pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index fd9bd9bd41..3831b911cf 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -18,7 +18,6 @@ use crate::{ resources::{Resource, ResourceType}, users::{send_email_if_possible, WorkspaceInvite, VALID_USERNAME}, utils::require_super_admin, - variables::build_crypt, webhook_util::{InstanceEvent, WebhookShared}, }; #[cfg(feature = "stripe")] @@ -43,6 +42,7 @@ use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; use windmill_common::schedule::Schedule; use windmill_common::users::username_to_permissioned_as; +use windmill_common::variables::build_crypt; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::workspaces::WorkspaceGitRepo; use windmill_common::{ diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 981b4989e1..7b90a32b60 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -50,4 +50,5 @@ git-version.workspace = true cron.workspace = true tracing-loki = { version = "^0", optional = true } aws-sdk-s3.workspace = true -aws-config.workspace = true \ No newline at end of file +aws-config.workspace = true +magic-crypt.workspace = true diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 0a65696ee2..15799d17e4 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -6,9 +6,15 @@ * LICENSE-AGPL for a copy of the license. */ +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use serde::{Deserialize, Serialize}; +use sqlx::{Postgres, Transaction}; -use crate::BASE_URL; +use crate::{BASE_URL, DB}; + +lazy_static::lazy_static! { + pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); +} #[derive(Serialize, Clone)] @@ -60,6 +66,72 @@ pub struct CreateVariable { pub is_oauth: Option, } +pub async fn build_crypt<'c>( + db: &mut Transaction<'c, Postgres>, + w_id: &str, +) -> crate::error::Result { + let key = get_workspace_key(w_id, db).await?; + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", key, salt) + } else { + key + }; + Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) +} + +pub async fn get_workspace_key<'c>( + w_id: &str, + db: &mut Transaction<'c, Postgres>, +) -> crate::error::Result { + let key = sqlx::query_scalar!( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", + w_id + ) + .fetch_one(&mut **db) + .await + .map_err(|e| crate::Error::InternalErr(format!("fetching workspace key: {e}")))?; + Ok(key) +} + +pub async fn get_secret_value_as_admin( + db: &DB, + w_id: &str, + path: &str, +) -> crate::error::Result { + let variable_o = sqlx::query!( + "SELECT value, is_secret, path from variable WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id + ) + .fetch_optional(db) + .await?; + + let variable = if let Some(variable) = variable_o { + variable + } else { + return Err(crate::Error::NotFound(format!( + "variable {} not found in workspace {}", + path, w_id + ))); + }; + + let r = if variable.is_secret { + let value = variable.value; + if !value.is_empty() { + let mut tx = db.begin().await?; + let mc = build_crypt(&mut tx, &w_id).await?; + tx.commit().await?; + + mc.decrypt_base64_to_string(value) + .map_err(|e| crate::Error::InternalErr(e.to_string()))? + } else { + "".to_string() + } + } else { + variable.value + }; + + Ok(r) +} + pub async fn get_reserved_variables( w_id: &str, token: &str, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index db26a4cedf..483f27097c 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -16,6 +16,7 @@ use windmill_common::{ error::{self, Error}, jobs::QueuedJob, utils::calculate_hash, + variables::get_secret_value_as_admin, worker::WORKER_CONFIG, DB, }; @@ -107,6 +108,10 @@ pub async fn pip_compile( } else { requirements.to_string() }; + + #[cfg(feature = "enterprise")] + let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; + let req_hash = format!("py-{}", calculate_hash(&requirements)); if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", @@ -518,6 +523,38 @@ if args["{name}"] is None: )) } +async fn replace_pip_secret( + db: &DB, + w_id: &str, + req: &str, + worker_name: &str, + job_id: &Uuid, +) -> error::Result { + tracing::error!("FOO"); + if PIP_SECRET_VARIABLE.is_match(req) { + let capture = PIP_SECRET_VARIABLE.captures(req); + let variable = capture.unwrap().get(1).unwrap().as_str(); + if !variable.contains("/PIP_SECRET_") { + return Err(error::Error::InternalErr(format!( + "invalid secret variable in pip requirements, (last part of path ma): {}", + req + ))); + } + let secret = get_secret_value_as_admin(db, w_id, variable).await?; + tracing::info!( + worker_name = %worker_name, + job_id = %job_id, + workspace_id = %w_id, + "found secret variable in pip requirements: {}", + req + ); + let req = PIP_SECRET_VARIABLE.replace(req, secret.as_str()); + Ok(req.to_string()) + } else { + Ok(req.to_string()) + } +} + async fn handle_python_deps( job_dir: &str, requirements_o: Option, @@ -597,6 +634,10 @@ async fn handle_python_deps( Ok(additional_python_paths) } +lazy_static::lazy_static! { + static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^s\}]+)\}").unwrap(); +} + pub async fn handle_python_reqs( requirements: Vec<&str>, job_id: &Uuid, @@ -705,7 +746,7 @@ pub async fn handle_python_reqs( .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let fssafe_req = NON_ALPHANUM_CHAR.replace_all(req, "_").to_string(); + let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string(); let req = format!("'{}'", req); let mut command_args = vec![ PYTHON_PATH.as_str(),