starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add ability to use secrets in pip requirements

Browse Source
This commit is contained in:
Ruben Fiszel
2024-01-10 13:12:13 +01:00
parent 45a794b1b3
commit ca4aac5eaa
10 changed files with 176 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+1
View File
@@ -9416,6 +9416,7 @@ dependencies = [
"hyper",
"itertools 0.12.0",
"lazy_static",
"magic-crypt",
"prometheus",
"rand 0.8.5",
"regex",
backend/windmill-api/src/apps.rs
+1 -1
View File
@@ -10,7 +10,6 @@ use std::collections::HashMap;
use crate::{
db::{ApiAuthed, DB},
users::{require_owner_of_path, OptAuthed},
variables::build_crypt,
webhook_util::{WebhookMessage, WebhookShared},
HTTP_CLIENT,
};
@@ -39,6 +38,7 @@ use windmill_common::{
utils::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
},
variables::build_crypt,
};
use windmill_queue::{push, PushArgs, PushIsolationLevel, QueueTransaction};
backend/windmill-api/src/jobs.rs
+1 -1
View File
@@ -10,6 +10,7 @@ use axum::http::HeaderValue;
use serde_json::value::RawValue;
use std::collections::HashMap;
use windmill_common::flow_status::RestartedFrom;
use windmill_common::variables::get_workspace_key;
use crate::db::ApiAuthed;
@@ -17,7 +18,6 @@ use crate::{
db::DB,
users::{check_scopes, require_owner_of_path, OptAuthed},
utils::require_super_admin,
variables::get_workspace_key,
};
use anyhow::Context;
use axum::{
backend/windmill-api/src/oauth2.rs
+2 -5
View File
@@ -38,16 +38,13 @@ use windmill_common::jobs::JobPayload;
use windmill_common::more_serde::maybe_number_opt;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::utils::{not_found_if_none, now_from_db};
use windmill_common::variables::build_crypt;
use crate::db::ApiAuthed;
use crate::saml::SamlSsoLogin;
use crate::users::{login_externally, LoginUserInfo};
use crate::webhook_util::{InstanceEvent, WebhookShared};
use crate::{
db::DB,
variables::{build_crypt, encrypt},
workspaces::WorkspaceSettings,
};
use crate::{db::DB, variables::encrypt, workspaces::WorkspaceSettings};
use crate::{BASE_URL, HTTP_CLIENT, IS_SECURE, OAUTH_CLIENTS, SLACK_SIGNING_SECRET};
use windmill_common::error::{self, to_anyhow, Error};
use windmill_common::oauth2::*;
backend/windmill-api/src/openai.rs
+4 -2
View File
@@ -2,7 +2,6 @@ use std::collections::HashMap;
use crate::{
db::{ApiAuthed, DB},
variables::build_crypt,
HTTP_CLIENT,
};
@@ -17,7 +16,10 @@ use magic_crypt::MagicCryptTrait;
use quick_cache::sync::Cache;
use serde_json::value::RawValue;
use windmill_audit::{audit_log, ActionKind};
use windmill_common::error::{to_anyhow, Error};
use windmill_common::{
error::{to_anyhow, Error},
variables::build_crypt,
};
use serde::Deserialize;
backend/windmill-api/src/variables.rs
+49 -74
View File
@@ -25,7 +25,9 @@ use windmill_common::{
db::UserDB,
error::{Error, JsonResult, Result},
utils::{not_found_if_none, StripPath},
variables::{get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable},
variables::{
build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable,
},
};
use lazy_static::lazy_static;
@@ -192,57 +194,6 @@ async fn get_value(
.map(Json);
}
pub async fn get_value_internal<'c>(
mut tx: Transaction<'c, Postgres>,
db: &DB,
w_id: &str,
path: &str,
username: &str,
) -> Result<String> {
let variable_o = sqlx::query!(
"SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable
LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id
)
.fetch_optional(&mut *tx)
.await?;
let variable = if let Some(variable) = variable_o {
variable
} else {
explain_variable_perm_error(path, w_id, db).await?;
unreachable!()
};
let r = if variable.is_secret {
audit_log(
&mut *tx,
username,
"variables.decrypt_secret",
ActionKind::Execute,
&w_id,
Some(&variable.path),
None,
)
.await?;
let value = variable.value;
if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
_refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await?
} else if !value.is_empty() {
let mc = build_crypt(&mut tx, &w_id).await?;
tx.commit().await?;
mc.decrypt_base64_to_string(value)
.map_err(|e| Error::InternalErr(e.to_string()))?
} else {
"".to_string()
}
} else {
variable.value
};
Ok(r)
}
async fn explain_variable_perm_error(
path: &str,
w_id: &str,
@@ -604,31 +555,55 @@ fn replace_path(v: serde_json::Value, path: &str, npath: &str) -> Value {
}
}
pub async fn build_crypt<'c>(
db: &mut Transaction<'c, Postgres>,
pub async fn get_value_internal<'c>(
mut tx: Transaction<'c, Postgres>,
db: &DB,
w_id: &str,
) -> Result<MagicCrypt256> {
let key = get_workspace_key(w_id, db).await?;
let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() {
format!("{}{}", key, salt)
} else {
key
};
Ok(magic_crypt::new_magic_crypt!(crypt_key, 256))
}
pub async fn get_workspace_key<'c>(
w_id: &str,
db: &mut Transaction<'c, Postgres>,
path: &str,
username: &str,
) -> Result<String> {
let key = sqlx::query_scalar!(
"SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'",
w_id
let variable_o = sqlx::query!(
"SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable
LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id
)
.fetch_one(&mut **db)
.await
.map_err(|e| Error::InternalErr(format!("fetching workspace key: {e}")))?;
Ok(key)
.fetch_optional(&mut *tx)
.await?;
let variable = if let Some(variable) = variable_o {
variable
} else {
explain_variable_perm_error(path, w_id, db).await?;
unreachable!()
};
let r = if variable.is_secret {
audit_log(
&mut *tx,
username,
"variables.decrypt_secret",
ActionKind::Execute,
&w_id,
Some(&variable.path),
None,
)
.await?;
let value = variable.value;
if variable.is_expired.unwrap_or(false) && variable.account.is_some() {
_refresh_token(tx, &variable.path, &w_id, variable.account.unwrap()).await?
} else if !value.is_empty() {
let mc = build_crypt(&mut tx, &w_id).await?;
tx.commit().await?;
mc.decrypt_base64_to_string(value)
.map_err(|e| Error::InternalErr(e.to_string()))?
} else {
"".to_string()
}
} else {
variable.value
};
Ok(r)
}
pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String {
backend/windmill-api/src/workspaces.rs
+1 -1
View File
@@ -18,7 +18,6 @@ use crate::{
resources::{Resource, ResourceType},
users::{send_email_if_possible, WorkspaceInvite, VALID_USERNAME},
utils::require_super_admin,
variables::build_crypt,
webhook_util::{InstanceEvent, WebhookShared},
};
#[cfg(feature = "stripe")]
@@ -43,6 +42,7 @@ use windmill_common::db::UserDB;
use windmill_common::s3_helpers::LargeFileStorage;
use windmill_common::schedule::Schedule;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::variables::build_crypt;
use windmill_common::worker::CLOUD_HOSTED;
use windmill_common::workspaces::WorkspaceGitRepo;
use windmill_common::{
backend/windmill-common/Cargo.toml
+2 -1
View File
@@ -50,4 +50,5 @@ git-version.workspace = true
cron.workspace = true
tracing-loki = { version = "^0", optional = true }
aws-sdk-s3.workspace = true
aws-config.workspace = true
aws-config.workspace = true
magic-crypt.workspace = true
backend/windmill-common/src/variables.rs
+73 -1
View File
@@ -6,9 +6,15 @@
* LICENSE-AGPL for a copy of the license.
*/
use magic_crypt::{MagicCrypt256, MagicCryptTrait};
use serde::{Deserialize, Serialize};
use sqlx::{Postgres, Transaction};
use crate::BASE_URL;
use crate::{BASE_URL, DB};
lazy_static::lazy_static! {
pub static ref SECRET_SALT: Option<String> = std::env::var("SECRET_SALT").ok();
}
#[derive(Serialize, Clone)]
@@ -60,6 +66,72 @@ pub struct CreateVariable {
pub is_oauth: Option<bool>,
}
pub async fn build_crypt<'c>(
db: &mut Transaction<'c, Postgres>,
w_id: &str,
) -> crate::error::Result<MagicCrypt256> {
let key = get_workspace_key(w_id, db).await?;
let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() {
format!("{}{}", key, salt)
} else {
key
};
Ok(magic_crypt::new_magic_crypt!(crypt_key, 256))
}
pub async fn get_workspace_key<'c>(
w_id: &str,
db: &mut Transaction<'c, Postgres>,
) -> crate::error::Result<String> {
let key = sqlx::query_scalar!(
"SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'",
w_id
)
.fetch_one(&mut **db)
.await
.map_err(|e| crate::Error::InternalErr(format!("fetching workspace key: {e}")))?;
Ok(key)
}
pub async fn get_secret_value_as_admin(
db: &DB,
w_id: &str,
path: &str,
) -> crate::error::Result<String> {
let variable_o = sqlx::query!(
"SELECT value, is_secret, path from variable WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id
)
.fetch_optional(db)
.await?;
let variable = if let Some(variable) = variable_o {
variable
} else {
return Err(crate::Error::NotFound(format!(
"variable {} not found in workspace {}",
path, w_id
)));
};
let r = if variable.is_secret {
let value = variable.value;
if !value.is_empty() {
let mut tx = db.begin().await?;
let mc = build_crypt(&mut tx, &w_id).await?;
tx.commit().await?;
mc.decrypt_base64_to_string(value)
.map_err(|e| crate::Error::InternalErr(e.to_string()))?
} else {
"".to_string()
}
} else {
variable.value
};
Ok(r)
}
pub async fn get_reserved_variables(
w_id: &str,
token: &str,
backend/windmill-worker/src/python_executor.rs
+42 -1
View File
@@ -16,6 +16,7 @@ use windmill_common::{
error::{self, Error},
jobs::QueuedJob,
utils::calculate_hash,
variables::get_secret_value_as_admin,
worker::WORKER_CONFIG,
DB,
};
@@ -107,6 +108,10 @@ pub async fn pip_compile(
} else {
requirements.to_string()
};
#[cfg(feature = "enterprise")]
let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?;
let req_hash = format!("py-{}", calculate_hash(&requirements));
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
@@ -518,6 +523,38 @@ if args["{name}"] is None:
))
}
async fn replace_pip_secret(
db: &DB,
w_id: &str,
req: &str,
worker_name: &str,
job_id: &Uuid,
) -> error::Result<String> {
tracing::error!("FOO");
if PIP_SECRET_VARIABLE.is_match(req) {
let capture = PIP_SECRET_VARIABLE.captures(req);
let variable = capture.unwrap().get(1).unwrap().as_str();
if !variable.contains("/PIP_SECRET_") {
return Err(error::Error::InternalErr(format!(
"invalid secret variable in pip requirements, (last part of path ma): {}",
req
)));
}
let secret = get_secret_value_as_admin(db, w_id, variable).await?;
tracing::info!(
worker_name = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
"found secret variable in pip requirements: {}",
req
);
let req = PIP_SECRET_VARIABLE.replace(req, secret.as_str());
Ok(req.to_string())
} else {
Ok(req.to_string())
}
}
async fn handle_python_deps(
job_dir: &str,
requirements_o: Option<String>,
@@ -597,6 +634,10 @@ async fn handle_python_deps(
Ok(additional_python_paths)
}
lazy_static::lazy_static! {
static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^s\}]+)\}").unwrap();
}
pub async fn handle_python_reqs(
requirements: Vec<&str>,
job_id: &Uuid,
@@ -705,7 +746,7 @@ pub async fn handle_python_reqs(
.stderr(Stdio::piped());
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
} else {
let fssafe_req = NON_ALPHANUM_CHAR.replace_all(req, "_").to_string();
let fssafe_req = NON_ALPHANUM_CHAR.replace_all(&req, "_").to_string();
let req = format!("'{}'", req);
let mut command_args = vec![
PYTHON_PATH.as_str(),