diff --git a/backend/migrations/20250322171903_workspace_envs_cache.down.sql b/backend/migrations/20250322171903_workspace_envs_cache.down.sql new file mode 100644 index 0000000000..68a678236d --- /dev/null +++ b/backend/migrations/20250322171903_workspace_envs_cache.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP TRIGGER workspace_envs_change_trigger ON workspace_env; +DROP FUNCTION notify_workspace_envs_change(); diff --git a/backend/migrations/20250322171903_workspace_envs_cache.up.sql b/backend/migrations/20250322171903_workspace_envs_cache.up.sql new file mode 100644 index 0000000000..439a4dd106 --- /dev/null +++ b/backend/migrations/20250322171903_workspace_envs_cache.up.sql @@ -0,0 +1,15 @@ +-- Add up migration script here +-- Add up migration script here + +CREATE OR REPLACE FUNCTION notify_workspace_envs_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_workspace_envs_change', NEW.workspace_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER workspace_envs_change_trigger +AFTER INSERT OR UPDATE OF name, value OR DELETE ON workspace_env +FOR EACH ROW +EXECUTE FUNCTION notify_workspace_envs_change(); diff --git a/backend/src/main.rs b/backend/src/main.rs index 235365926b..c349a87249 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -702,6 +702,11 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id); windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id); }, + "notify_workspace_envs_change" => { + let workspace_id = n.payload(); + tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", workspace_id); + windmill_common::variables::CUSTOM_ENVS_CACHE.remove(workspace_id); + }, "notify_global_setting_change" => { tracing::info!("Global setting change detected: {}", n.payload()); match n.payload() { @@ -980,6 +985,7 @@ async fn listen_pg(url: &str) -> Option { "notify_config_change", "notify_global_setting_change", "notify_webhook_change", + "notify_workspace_envs_change", ]) .await { diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 1817945705..90baec78a5 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -10,6 +10,7 @@ use crate::error; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; +use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; lazy_static::lazy_static! { @@ -160,6 +161,10 @@ pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { pub const WM_SCHEDULED_FOR: &str = "WM_SCHEDULED_FOR"; +lazy_static::lazy_static! { + pub static ref CUSTOM_ENVS_CACHE: Cache)> = Cache::new(100); +} + pub async fn get_reserved_variables( db: &DB, w_id: &str, @@ -201,6 +206,8 @@ pub async fn get_reserved_variables( } }; + let custom_envs = get_cached_workspace_envs(db, w_id).await; + let joined_schedule_path = schedule_path .clone() .unwrap_or("manual".to_string()) @@ -223,133 +230,157 @@ pub async fn get_reserved_variables( }; vec![ - ContextualVariable { - name: "WM_WORKSPACE".to_string(), - value: w_id.to_string(), - description: "Workspace id of the current script".to_string(), + ContextualVariable { + name: "WM_WORKSPACE".to_string(), + value: w_id.to_string(), + description: "Workspace id of the current script".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_TOKEN".to_string(), + value: token.to_string(), + description: "Token ephemeral to the current script with equal permission to the \ + permission of the run (Usable as a bearer token)" + .to_string(), is_custom: false, - }, - ContextualVariable { - name: "WM_TOKEN".to_string(), - value: token.to_string(), - description: "Token ephemeral to the current script with equal permission to the \ - permission of the run (Usable as a bearer token)" - .to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_EMAIL".to_string(), - value: email.to_string(), - description: "Email of the user that executed the current script".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_USERNAME".to_string(), - value: username.to_string(), - description: "Username of the user that executed the current script".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_BASE_URL".to_string(), - value: BASE_URL.read().await.clone(), - description: "base url of this instance".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_JOB_ID".to_string(), - value: job_id.to_string(), - description: "Job id of the current script".to_string(), - is_custom: false, - }, - ContextualVariable { - name: WM_SCHEDULED_FOR.to_string(), - value: scheduled_for - .map(|ts| ts.to_rfc3339_opts(SecondsFormat::Secs, true)) - .unwrap_or_else(|| "".to_string()), - description: "date-time in UTC (e.g: 2014-11-28T12:45:59.324310806Z) of when the job was scheduled".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_JOB_PATH".to_string(), - value: path.unwrap_or_else(|| "".to_string()), - description: "Path of the script or flow being run if any".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_FLOW_JOB_ID".to_string(), - value: flow_id.unwrap_or_else(|| "".to_string()), - description: "Job id of the encapsulating flow if the job is a flow step".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_ROOT_FLOW_JOB_ID".to_string(), - value: root_flow_id.unwrap_or_else(|| "".to_string()), - description: "Job id of the root flow if the job is a flow step".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_FLOW_PATH".to_string(), - value: flow_path.unwrap_or_else(|| "".to_string()), - description: "Path of the encapsulating flow if the job is a flow step".to_string(), - is_custom: false, - }, + }, + ContextualVariable { + name: "WM_EMAIL".to_string(), + value: email.to_string(), + description: "Email of the user that executed the current script".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_USERNAME".to_string(), + value: username.to_string(), + description: "Username of the user that executed the current script".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_BASE_URL".to_string(), + value: BASE_URL.read().await.clone(), + description: "base url of this instance".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_JOB_ID".to_string(), + value: job_id.to_string(), + description: "Job id of the current script".to_string(), + is_custom: false, + }, + ContextualVariable { + name: WM_SCHEDULED_FOR.to_string(), + value: scheduled_for + .map(|ts| ts.to_rfc3339_opts(SecondsFormat::Secs, true)) + .unwrap_or_else(|| "".to_string()), + description: "date-time in UTC (e.g: 2014-11-28T12:45:59.324310806Z) of when the job was scheduled".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_JOB_PATH".to_string(), + value: path.unwrap_or_else(|| "".to_string()), + description: "Path of the script or flow being run if any".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_FLOW_JOB_ID".to_string(), + value: flow_id.unwrap_or_else(|| "".to_string()), + description: "Job id of the encapsulating flow if the job is a flow step".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_ROOT_FLOW_JOB_ID".to_string(), + value: root_flow_id.unwrap_or_else(|| "".to_string()), + description: "Job id of the root flow if the job is a flow step".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_FLOW_PATH".to_string(), + value: flow_path.unwrap_or_else(|| "".to_string()), + description: "Path of the encapsulating flow if the job is a flow step".to_string(), + is_custom: false, + }, - ContextualVariable { - name: "WM_SCHEDULE_PATH".to_string(), - value: schedule_path.unwrap_or_else(|| "".to_string()), - description: "Path of the schedule if the job of the step or encapsulating step has \ - been triggered by a schedule" - .to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_PERMISSIONED_AS".to_string(), - value: permissioned_as.to_string(), - description: "Fully Qualified (u/g) owner name of executor of the job".to_string(), + ContextualVariable { + name: "WM_SCHEDULE_PATH".to_string(), + value: schedule_path.unwrap_or_else(|| "".to_string()), + description: "Path of the schedule if the job of the step or encapsulating step has \ + been triggered by a schedule" + .to_string(), is_custom: false, - }, - ContextualVariable { - name: "WM_STATE_PATH".to_string(), - value: state_path.clone(), - description: "State resource path unique to a script and its trigger".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_FLOW_STEP_ID".to_string(), - value: step_id.unwrap_or_else(|| "".to_string()), - description: "The node id in a flow (like 'a', 'b', or 'f')".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_OBJECT_PATH".to_string(), - value: object_path, - description: "Script or flow step execution unique path, useful for storing results in an external service".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_OIDC_JWT".to_string(), - value: jwt_token.unwrap_or_else(|| "".to_string()), - description: "OIDC JWT token (EE only)".to_string(), - is_custom: false, - }, - ContextualVariable { - name: "WM_WORKER_GROUP".to_string(), - value: WORKER_GROUP.clone(), - description: "name of the worker group the job is running on".to_string(), - is_custom: false, - }, - ].into_iter().chain( sqlx::query_as::<_, (String, String)>( - "SELECT name, value FROM workspace_env WHERE workspace_id = $1", - ) - .bind(w_id) - .fetch_all(db) - .await - .unwrap_or_default() - .into_iter().map(|(name, value)| ContextualVariable { - name, - value, - description: "Custom workspace environment variable".to_string(), - is_custom: true, - })).collect() + }, + ContextualVariable { + name: "WM_PERMISSIONED_AS".to_string(), + value: permissioned_as.to_string(), + description: "Fully Qualified (u/g) owner name of executor of the job".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_STATE_PATH".to_string(), + value: state_path.clone(), + description: "State resource path unique to a script and its trigger".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_FLOW_STEP_ID".to_string(), + value: step_id.unwrap_or_else(|| "".to_string()), + description: "The node id in a flow (like 'a', 'b', or 'f')".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_OBJECT_PATH".to_string(), + value: object_path, + description: "Script or flow step execution unique path, useful for storing results in an external service".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_OIDC_JWT".to_string(), + value: jwt_token.unwrap_or_else(|| "".to_string()), + description: "OIDC JWT token (EE only)".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_WORKER_GROUP".to_string(), + value: WORKER_GROUP.clone(), + description: "name of the worker group the job is running on".to_string(), + is_custom: false, + }, +].into_iter().chain(custom_envs.into_iter().map(|(name, value)| ContextualVariable { + name, + value, + description: "Custom workspace environment variable".to_string(), + is_custom: true, +}) +).collect() } +async fn get_cached_workspace_envs( + db: &sqlx::Pool, + w_id: &str, +) -> Vec<(String, String)> { + let cached_envs_o = CUSTOM_ENVS_CACHE.get(w_id).and_then(|(ts, envs)| { + if ts > chrono::Utc::now().timestamp() - (60 * 15) { + Some(envs) + } else { + None + } + }); + + let custom_envs = if let Some(cached_envs) = cached_envs_o { + cached_envs + } else { + let custom_envs = sqlx::query_as::<_, (String, String)>( + "SELECT name, value FROM workspace_env WHERE workspace_id = $1", + ) + .bind(w_id) + .fetch_all(db) + .await + .unwrap_or_default(); + CUSTOM_ENVS_CACHE.insert( + w_id.to_string(), + (chrono::Utc::now().timestamp(), custom_envs.clone()), + ); + custom_envs + }; + custom_envs +} diff --git a/frontend/src/lib/components/ContextualVariableEditor.svelte b/frontend/src/lib/components/ContextualVariableEditor.svelte index 4e1dfcbddf..26c36211da 100644 --- a/frontend/src/lib/components/ContextualVariableEditor.svelte +++ b/frontend/src/lib/components/ContextualVariableEditor.svelte @@ -40,9 +40,17 @@ name: name } }) - sendUserToast(`${edit ? 'Updated' : 'Created'} contextual variable ${name}`) + sendUserToast( + `${ + edit ? 'Updated' : 'Created' + } contextual variable ${name}. It may take up to a few minutes to update.` + ) dispatch('update') + drawer.closeDrawer() + setTimeout(() => { + dispatch('update') + }, 5000) } diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 5219fa3729..f4d483d4e6 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -126,7 +126,12 @@ } }) loadContextualVariables() - sendUserToast(`Custom contextual variable ${row.name} was deleted`) + sendUserToast( + `Custom contextual variable ${row.name} was deleted. It may take up to a few minutes to update.` + ) + setTimeout(() => { + loadContextualVariables() + }, 5000) } @@ -139,327 +144,332 @@ f={(x) => x.path + ' ' + x.description} /> -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.variables} - +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.variables} + {:else} - - -
- {#if tab == 'contextual' && ($userStore?.is_admin || $userStore?.is_super_admin)} - - {:else} - - {/if} -
- - - - - { - loadVariables() - }} - /> - - - -
- - Workspace + + +
+ {#if tab == 'contextual' && ($userStore?.is_admin || $userStore?.is_super_admin)} + + {:else} + + {/if}
- - -
- - Contextual - - Contextual variables are passed as environment variables when running a script and depends - on the execution context. - -
- - - {#if tab == 'workspace'} -
- -
-
- -
-
- {#if !filteredItems} - - {#each new Array(3) as _} - - {/each} - {:else if filteredItems.length == 0} -
-
No variables found
-
- Try changing the filters or creating a new variable -
+ + + + + { + loadVariables() + }} + /> + + + +
+ + Workspace
- {:else} - - - - - Path - Value - Description - - - - - - {#each filteredItems as { path, value, is_secret, description, extra_perms, canWrite, account, is_refreshed, is_expired, refresh_error, is_linked, marked }} - - - - - - variableEditor.editVariable(path)} - href="#{path}" - > - {#if marked} - {@html marked} - {:else} - {path} - {/if} - - - - -
- {#if value} - {truncate(value, 20)} - {:else} - ∗∗∗∗ - {/if} -
- {#if is_secret} - - - This item is secret - - {/if} - - - - {truncate(description ?? '', 50)} - - - -
- {#if is_linked} - - -
- This variable is linked with a resource of the same path. They are deleted - and renamed together. -
- - {/if} - {#if account} - - -
- This OAuth token will be kept up-to-date in the background by Windmill - using its refresh token -
- - {/if} - - {#if is_refreshed} -
- {#if refresh_error} - -
- - -
- -
- Latest exchange of the refresh token did not succeed. Error: {refresh_error} -
- - {:else if is_expired} - - -
- The access_token is expired, it will get renewed the next time this - variable is fetched or you can request is to be refreshed in the - dropdown on the right. -
- - {:else} - - -
- The variable was connected through OAuth and the token is not expired. -
- - {/if} -
- {/if} -
- - - { - let owner = isOwner(path, $userStore, $workspaceStore) - return [ - { - displayName: 'Edit', - icon: Pen, - action: () => variableEditor.editVariable(path), - disabled: !canWrite - }, - { - displayName: 'Delete', - icon: Trash, - type: 'delete', - action: (event) => { - if (event['shiftKey']) { - deleteVariable(path, account) - } else { - deleteConfirmedCallback = () => { - deleteVariable(path, account) - } - } - }, - disabled: !owner - }, - ...(isDeployable(is_secret ? 'secret' : 'variable', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer.openDrawer(path, 'variable') - } - } - ] - : []), - { - displayName: owner ? 'Share' : 'See Permissions', - action: () => { - shareModal.openDrawer(path, 'variable') - }, - icon: Share - }, - ...(account != undefined - ? [ - { - displayName: 'Refresh token', - icon: RefreshCw, - action: async () => { - await OauthService.refreshToken({ - workspace: $workspaceStore ?? '', - id: account ?? 0, - requestBody: { - path - } - }) - sendUserToast('Token refreshed') - loadVariables() - } - } - ] - : []) - ] - }} - /> - - - {/each} - - - {/if} -
- {:else if tab == 'contextual'} -
- {#if loading.contextual} - - {#each new Array(8) as _} - - {/each} - {:else} - - {#if contextualVariables.filter((x) => x.is_custom).length === 0} + + +
+ + Contextual + + Contextual variables are passed as environment variables when running a script and + depends on the execution context. + +
+ + + {#if tab == 'workspace'} +
+ +
+
+ +
+
+ {#if !filteredItems} + + {#each new Array(3) as _} + + {/each} + {:else if filteredItems.length == 0}
-
No custom contextual variables found
+
No variables found
+
+ Try changing the filters or creating a new variable +
{:else} - x.is_custom)} - keys={['name', 'value']} - getRowActions={$userStore?.is_admin || $userStore?.is_super_admin - ? (row) => { - return [ - { - displayName: 'Edit', - action: () => contextualVariableEditor.editVariable(row.name, row.value) - }, - { - displayName: 'Delete', - type: 'delete', - action: () => { - deleteContextualVariable(row) + + + + + Path + Value + Description + + + + + + {#each filteredItems as { path, value, is_secret, description, extra_perms, canWrite, account, is_refreshed, is_expired, refresh_error, is_linked, marked }} + + + + + + variableEditor.editVariable(path)} + href="#{path}" + > + {#if marked} + {@html marked} + {:else} + {path} + {/if} + + + + +
+ {#if value} + {truncate(value, 20)} + {:else} + ∗∗∗∗ + {/if} +
+ {#if is_secret} + + + This item is secret + + {/if} + + + + {truncate(description ?? '', 50)} + + + +
+ {#if is_linked} + + +
+ This variable is linked with a resource of the same path. They are + deleted and renamed together. +
+ + {/if} + {#if account} + + +
+ This OAuth token will be kept up-to-date in the background by Windmill + using its refresh token +
+ + {/if} + + {#if is_refreshed} +
+ {#if refresh_error} + +
+ + +
+ +
+ Latest exchange of the refresh token did not succeed. Error: {refresh_error} +
+ + {:else if is_expired} + + +
+ The access_token is expired, it will get renewed the next time this + variable is fetched or you can request is to be refreshed in the + dropdown on the right. +
+ + {:else} + + +
+ The variable was connected through OAuth and the token is not + expired. +
+ + {/if} +
+ {/if} +
+ + + { + let owner = isOwner(path, $userStore, $workspaceStore) + return [ + { + displayName: 'Edit', + icon: Pen, + action: () => variableEditor.editVariable(path), + disabled: !canWrite + }, + { + displayName: 'Delete', + icon: Trash, + type: 'delete', + action: (event) => { + if (event['shiftKey']) { + deleteVariable(path, account) + } else { + deleteConfirmedCallback = () => { + deleteVariable(path, account) + } + } + }, + disabled: !owner + }, + ...(isDeployable( + is_secret ? 'secret' : 'variable', + path, + deployUiSettings + ) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer.openDrawer(path, 'variable') + } + } + ] + : []), + { + displayName: owner ? 'Share' : 'See Permissions', + action: () => { + shareModal.openDrawer(path, 'variable') + }, + icon: Share + }, + ...(account != undefined + ? [ + { + displayName: 'Refresh token', + icon: RefreshCw, + action: async () => { + await OauthService.refreshToken({ + workspace: $workspaceStore ?? '', + id: account ?? 0, + requestBody: { + path + } + }) + sendUserToast('Token refreshed') + loadVariables() + } + } + ] + : []) + ] + }} + /> + + + {/each} + + + {/if} +
+ {:else if tab == 'contextual'} +
+ {#if loading.contextual} + + {#each new Array(8) as _} + + {/each} + {:else} + + {#if contextualVariables.filter((x) => x.is_custom).length === 0} +
+
No custom contextual variables found
+
+ {:else} + x.is_custom)} + keys={['name', 'value']} + getRowActions={$userStore?.is_admin || $userStore?.is_super_admin + ? (row) => { + return [ + { + displayName: 'Edit', + action: () => contextualVariableEditor.editVariable(row.name, row.value) + }, + { + displayName: 'Delete', + type: 'delete', + action: () => { + deleteContextualVariable(row) + } } - } - ] - } - : undefined} + ] + } + : undefined} + /> + {/if} + + !x.is_custom)} + keys={['name', 'value', 'description']} /> {/if} - - !x.is_custom)} - keys={['name', 'value', 'description']} - /> - {/if} -
- {/if} - +
+ {/if} + {/if}