From 8c47957a027bae7698e45160db67d23c8783ebdd Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 5 Jun 2023 17:42:57 +0200 Subject: [PATCH] feat: support custom env variables (#1675) * custom envs * feat: custom envs * only on non cloud * remove unecessary imports --- .../20230605043929_add_envs.down.sql | 2 + .../migrations/20230605043929_add_envs.up.sql | 2 + backend/sqlx-data.json | 105 +++++++++--------- backend/windmill-api/openapi.yaml | 8 ++ backend/windmill-api/src/scripts.rs | 18 ++- backend/windmill-common/src/scripts.rs | 3 + backend/windmill-worker/src/go_executor.rs | 4 +- .../windmill-worker/src/python_executor.rs | 4 +- backend/windmill-worker/src/worker.rs | 41 +++++-- .../src/lib/components/ScriptBuilder.svelte | 65 ++++++++++- .../display/AppFlowStatusComponent.svelte | 5 +- .../display/AppLogsComponent.svelte | 5 +- .../flows/content/ScriptEditorDrawer.svelte | 4 +- .../scripts/get/[...hash]/+page.svelte | 9 ++ 14 files changed, 199 insertions(+), 76 deletions(-) create mode 100644 backend/migrations/20230605043929_add_envs.down.sql create mode 100644 backend/migrations/20230605043929_add_envs.up.sql diff --git a/backend/migrations/20230605043929_add_envs.down.sql b/backend/migrations/20230605043929_add_envs.down.sql new file mode 100644 index 0000000000..dddc7c139d --- /dev/null +++ b/backend/migrations/20230605043929_add_envs.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE script DROP COLUMN envs; diff --git a/backend/migrations/20230605043929_add_envs.up.sql b/backend/migrations/20230605043929_add_envs.up.sql new file mode 100644 index 0000000000..c38fa59fc5 --- /dev/null +++ b/backend/migrations/20230605043929_add_envs.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE script ADD COLUMN envs VARCHAR(1000)[]; diff --git a/backend/sqlx-data.json b/backend/sqlx-data.json index ab5d80c0f8..a301b88ba5 100644 --- a/backend/sqlx-data.json +++ b/backend/sqlx-data.json @@ -3656,6 +3656,59 @@ }, "query": "SELECT EXISTS(SELECT 1 FROM variable WHERE path = $1 AND workspace_id = $2)" }, + "88a5f7d43e2775cdbc121e4f39f7d881af75c24740446ca86f26ba8b2e9e5405": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Varchar", + "Int8Array", + "Text", + "Text", + "Text", + "Varchar", + "Text", + "Bool", + "Jsonb", + "Text", + { + "Custom": { + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash" + ] + }, + "name": "script_lang" + } + }, + { + "Custom": { + "kind": { + "Enum": [ + "script", + "trigger", + "failure", + "command", + "approval" + ] + }, + "name": "script_kind" + } + }, + "Varchar", + "Bool", + "VarcharArray" + ] + } + }, + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17)" + }, "88b7589a6416da8be4b26af3bf30fcfcd6aeae7bc5a37e9a735cabbe2691c570": { "describe": { "columns": [ @@ -4010,58 +4063,6 @@ }, "query": "SELECT email, login_type::TEXT, super_admin, verified, name, company FROM password WHERE email = $1" }, - "90c5f1eea5d2cceb157ce0667a625f0fa4506961e2ad181765022a20b11d314d": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Left": [ - "Varchar", - "Int8", - "Varchar", - "Int8Array", - "Text", - "Text", - "Text", - "Varchar", - "Text", - "Bool", - "Jsonb", - "Text", - { - "Custom": { - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash" - ] - }, - "name": "script_lang" - } - }, - { - "Custom": { - "kind": { - "Enum": [ - "script", - "trigger", - "failure", - "command", - "approval" - ] - }, - "name": "script_kind" - } - }, - "Varchar", - "Bool" - ] - } - }, - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)" - }, "911b1e1f2a5ba6d5159916e5598020e680c45043b0736ad0153ee261a151dd90": { "describe": { "columns": [ diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 9420237d77..c431843a9c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5493,6 +5493,10 @@ components: type: boolean draft_only: type: boolean + envs: + type: array + items: + type: string required: - hash - path @@ -5540,6 +5544,10 @@ components: type: string draft_only: type: boolean + envs: + type: array + items: + type: string required: - path - summary diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 6dfdb6f984..b061da0b21 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -70,6 +70,8 @@ pub struct ScriptWDraft { pub schema: Option, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub envs: Option>, } pub fn global_service() -> Router { @@ -363,11 +365,18 @@ async fn create_script( }; let needs_lock_gen = lock.is_none(); + + let envs = ns.envs.as_ref().map(|x| x.as_slice()); + let envs = if ns.envs.is_none() || ns.envs.as_ref().unwrap().is_empty() { + None + } else { + envs + }; //::text::json is to ensure we use serde_json with preserve order sqlx::query!( "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, \ - content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16)", + content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17)", &w_id, &hash.0, ns.path, @@ -383,7 +392,8 @@ async fn create_script( ns.language: ScriptLang, ns.kind.unwrap_or(ScriptKind::Script): ScriptKind, ns.tag, - ns.draft_only + ns.draft_only, + envs ) .execute(&mut tx) .await?; @@ -550,7 +560,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, draft.value as draft FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, draft.value as draft FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 \ AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 4edab16bf8..9abedaf15c 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -139,6 +139,8 @@ pub struct Script { pub tag: Option, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub envs: Option>, } #[derive(Serialize)] @@ -189,6 +191,7 @@ pub struct NewScript { pub kind: Option, pub tag: Option, pub draft_only: Option, + pub envs: Option>, } #[derive(Deserialize)] diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 3efe04800e..11b2479f0c 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -1,4 +1,4 @@ -use std::process::Stdio; +use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; use tokio::{ @@ -40,6 +40,7 @@ pub async fn handle_go_job( shared_mount: &str, base_internal_url: &str, worker_name: &str, + envs: HashMap, ) -> Result { //go does not like executing modules at temp root let job_dir = &format!("{job_dir}/go"); @@ -210,6 +211,7 @@ func Run(req Req) (interface{{}}, error){{ let mut cmd = Command::new(GO_PATH.as_str()); cmd.current_dir(job_dir) .env_clear() + .envs(envs) .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 06b1921737..874c616851 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1,4 +1,4 @@ -use std::process::Stdio; +use std::{collections::HashMap, process::Stdio}; use itertools::Itertools; use regex::Regex; @@ -165,6 +165,7 @@ pub async fn handle_python_job( inner_content: &String, shared_mount: &str, base_internal_url: &str, + envs: HashMap, ) -> windmill_common::error::Result { create_dependencies_dir(job_dir).await; @@ -435,6 +436,7 @@ mount {{ Command::new(PYTHON_PATH.as_str()) .current_dir(job_dir) .env_clear() + .envs(envs) .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f0509ec1f3..4d942529fc 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1204,13 +1204,14 @@ async fn handle_code_execution_job( worker_name: &str ) -> error::Result { - let (inner_content, requirements_o, language) = match job.job_kind { + let (inner_content, requirements_o, language, envs) = match job.job_kind { JobKind::Preview => ( job.raw_code .clone() .unwrap_or_else(|| "no raw code".to_owned()), job.raw_lock.clone(), job.language.to_owned(), + None, ), JobKind::Script_Hub => { let script_path = job.script_path.clone().ok_or_else(|| Error::InternalErr(format!("expected script path for hub script")))?; @@ -1231,10 +1232,11 @@ async fn handle_code_execution_job( ( script.content, script.lockfile, - Some(script.language) + Some(script.language), + None )}, - JobKind::Script => sqlx::query_as::<_, (String, Option, Option)>( - "SELECT content, lock, language FROM script WHERE hash = $1 AND workspace_id = $2", + JobKind::Script => sqlx::query_as::<_, (String, Option, Option, Option>)>( + "SELECT content, lock, language, envs FROM script WHERE hash = $1 AND workspace_id = $2", ) .bind(&job.script_hash.unwrap_or(ScriptHash(0)).0) .bind(&job.workspace_id) @@ -1276,6 +1278,21 @@ mount {{ }; // println!("handle lang job {:?}", SystemTime::now()); + let envs = if *CLOUD_HOSTED || envs.is_none() { + HashMap::new() + } else { + let mut hm = HashMap::new(); + for s in envs.unwrap() { + let (k, v) = s.split_once('=').ok_or_else(|| { + Error::BadRequest(format!( + "Invalid env var: {}. Must be in the form of KEY=VALUE", + s + )) + })?; + hm.insert(k.to_string(), v.to_string()); + } + hm + }; let result: error::Result = match language { None => { @@ -1296,6 +1313,7 @@ mount {{ &inner_content, &shared_mount, base_internal_url, + envs ) .await } @@ -1308,7 +1326,8 @@ mount {{ job_dir, &inner_content, base_internal_url, - worker_name + worker_name, + envs ) .await } @@ -1323,7 +1342,8 @@ mount {{ requirements_o, &shared_mount, base_internal_url, - worker_name + worker_name, + envs ) .await } @@ -1337,7 +1357,8 @@ mount {{ job_dir, &shared_mount, base_internal_url, - worker_name + worker_name, + envs ) .await } @@ -1368,6 +1389,7 @@ async fn handle_bash_job( shared_mount: &str, base_internal_url: &str, worker_name: &str, + envs: HashMap, ) -> Result { logs.push_str("\n\n--- BASH CODE EXECUTION ---\n"); set_logs(logs, &job.id, db).await; @@ -1422,6 +1444,7 @@ async fn handle_bash_job( Command::new("/bin/bash") .current_dir(job_dir) .env_clear() + .envs(envs) .envs(reserved_variables) .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1469,7 +1492,8 @@ async fn handle_deno_job( job_dir: &str, inner_content: &String, base_internal_url: &str, - worker_name: &str + worker_name: &str, + envs: HashMap, ) -> error::Result { // let mut start = Instant::now(); @@ -1608,6 +1632,7 @@ run().catch(async (e) => {{ Command::new(DENO_PATH.as_str()) .current_dir(job_dir) .env_clear() + .envs(envs) .envs(reserved_variables) .envs(common_deno_proc_envs) .env("DENO_DIR", DENO_CACHE_DIR) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index d227e8f887..75bb9a2490 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -10,13 +10,13 @@ import ScriptEditor from './ScriptEditor.svelte' import ScriptSchema from './ScriptSchema.svelte' import { dirtyStore } from './common/confirmationModal/dirtyStore' - import { Badge, Button, Drawer, Kbd } from './common' - import { faSave } from '@fortawesome/free-solid-svg-icons' + import { Alert, Badge, Button, Drawer, Kbd } from './common' + import { faPlus, faSave } from '@fortawesome/free-solid-svg-icons' import LanguageIcon from './common/languageIcons/LanguageIcon.svelte' import type { SupportedLanguage } from '$lib/common' import Tooltip from './Tooltip.svelte' import DrawerContent from './common/drawer/DrawerContent.svelte' - import { Pen } from 'lucide-svelte' + import { Pen, X } from 'lucide-svelte' import autosize from 'svelte-autosize' import type Editor from './Editor.svelte' import { @@ -29,6 +29,8 @@ import { sendUserToast } from '$lib/toast' import { isCloudHosted } from '$lib/cloud' import Awareness from './Awareness.svelte' + import { Icon } from 'svelte-awesome' + import { fade } from 'svelte/transition' export let script: NewScript export let initialPath: string = '' @@ -157,7 +159,8 @@ is_template: script.is_template, language: script.language, kind: script.kind, - tag: script.tag + tag: script.tag, + envs: script.envs } }) history.replaceState(history.state, '', `/scripts/edit/${script.path}`) @@ -196,7 +199,8 @@ language: script.language, kind: script.kind, tag: script.tag, - draft_only: true + draft_only: true, + envs: script.envs } }) } @@ -402,6 +406,57 @@ {/if} {/if} + {#if !isCloudHosted()} +

Custom env variablesAdditional static custom env variables to pass to the script.

+
+ Format is: `{'='}` + {#if Array.isArray(script.envs ?? [])} + {#each script.envs ?? [] as v, i} +
+ + +
+ {/each} + {#if script.envs && script.envs.length > 0} +
+ Static envs variables are not passed in preview but solely on deployed scripts. + {/if} + {/if} +
+ {/if} +
+ +
diff --git a/frontend/src/lib/components/apps/components/display/AppFlowStatusComponent.svelte b/frontend/src/lib/components/apps/components/display/AppFlowStatusComponent.svelte index c87e29d67f..f3f3d07d93 100644 --- a/frontend/src/lib/components/apps/components/display/AppFlowStatusComponent.svelte +++ b/frontend/src/lib/components/apps/components/display/AppFlowStatusComponent.svelte @@ -10,7 +10,7 @@ export let id: string export let componentInput: AppInput | undefined - export let initializing: boolean | undefined = undefined + export let initializing: boolean | undefined = false export let customCss: ComponentCustomCSS<'flowstatuscomponent'> | undefined = undefined export let render: boolean @@ -21,6 +21,8 @@ loading: false }) + initializing = false + $: css = concatCustomCss($app.css?.flowstatuscomponent, customCss) let jobId: string | undefined @@ -34,7 +36,6 @@ {render} {componentInput} {id} - bind:initializing >
| undefined = undefined export let render: boolean @@ -23,6 +23,8 @@ loading: false }) + initializing = false + $: css = concatCustomCss($app.css?.logcomponent, customCss) let testJobLoader: TestJobLoader | undefined = undefined @@ -40,7 +42,6 @@ {render} {componentInput} {id} - bind:initializing >
+ {#if script.envs && script.envs.length > 0} +

Static Env Variables

+
    + {#each script.envs as e} +
  • {e}
    • + {/each} +
+ {/if} +

Webhooks