starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-20 08:01:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add workspace environment vars (custom contextual vars) (#3455)

Browse Source 
* feat: custom contextual variables

* chore: sqlx prepare

* fix: main merge
This commit is contained in:
HugoCasa
2024-03-23 11:40:55 +01:00
committed by GitHub
parent af58abd65d
commit 283d55008c
18 changed files with 358 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-58aa2e6de6cb9724750dae7405d664080e8d810278ff0dd52a2b8bfb7270fe44.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM workspace_env WHERE workspace_id = $1 AND name = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "58aa2e6de6cb9724750dae7405d664080e8d810278ff0dd52a2b8bfb7270fe44"
}
backend/.sqlx/query-73706be0610149682fa6131494212095e9494ade5e01b138c7fbbd16f42be791.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_env SET workspace_id = $1 WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "73706be0610149682fa6131494212095e9494ade5e01b138c7fbbd16f42be791"
}
backend/.sqlx/query-b8c66d905a6c7ffa6441c84b14ea897040069dac7367895813cc2d64a9867193.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "b8c66d905a6c7ffa6441c84b14ea897040069dac7367895813cc2d64a9867193"
}
backend/migrations/20240322164216_add_workspace_envs.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20240322164216_add_workspace_envs.up.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add up migration script here
CREATE TABLE workspace_env (
workspace_id varchar(50) not null,
name varchar(255) not null,
value varchar(1000) not null,
primary key (workspace_id, name)
)
backend/windmill-api/openapi.yaml
+32
View File
@@ -1841,6 +1841,35 @@ paths:
schema:
$ref: "#/components/schemas/WorkspaceDefaultScripts"
/w/{workspace}/workspaces/set_environment_variable:
post:
summary: set environment variable
operationId: setEnvironmentVariable
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: Workspace default app
required: true
content:
application/json:
schema:
type: object
properties:
name:
type: string
value:
type: string
required: [name]
responses:
"200":
description: status
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/encryption_key:
get:
summary: retrieves the encryption key for this workspace
@@ -8801,10 +8830,13 @@ components:
type: string
description:
type: string
is_custom:
type: boolean
required:
- name
- value
- description
- is_custom
CreateVariable:
type: object
backend/windmill-api/src/resources.rs
+2 -3
View File
@@ -385,9 +385,7 @@ async fn custom_component(
let cc = not_found_if_none(cc_o, "Custom Component", name)?;
let res = Response::builder().header(header::CONTENT_TYPE, "text/javascript");
Ok(res
.body(Body::from(cc))
.unwrap())
Ok(res.body(Body::from(cc)).unwrap())
}
#[derive(Deserialize)]
@@ -537,6 +535,7 @@ pub async fn transform_json_value<'c>(
};
let variables = variables::get_reserved_variables(
db,
&job.workspace_id,
token,
&job.email,
backend/windmill-api/src/variables.rs
+2
View File
@@ -57,9 +57,11 @@ pub fn workspaced_service() -> Router {
async fn list_contextual_variables(
Path(w_id): Path<String>,
ApiAuthed { username, email, .. }: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<Vec<ContextualVariable>> {
Ok(Json(
get_reserved_variables(
&db,
&w_id,
"q1A0qcPuO00yxioll7iph76N9CJDqn",
&email,
backend/windmill-api/src/workspaces.rs
+74
View File
@@ -104,6 +104,7 @@ pub fn workspaced_service() -> Router {
"/default_scripts",
post(edit_default_scripts).get(get_default_scripts),
)
.route("/set_environment_variable", post(set_environment_variable))
.route(
"/encryption_key",
get(get_encryption_key).post(set_encryption_key),
@@ -1200,6 +1201,71 @@ async fn edit_error_handler(
Ok(format!("Edit error_handler for workspace {}", &w_id))
}
#[derive(Deserialize)]
struct NewEnvironmentVariable {
name: String,
value: Option<String>,
}
async fn set_environment_variable(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(NewEnvironmentVariable { value, name }): Json<NewEnvironmentVariable>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = db.begin().await?;
match value {
Some(value) => {
sqlx::query!(
"INSERT INTO workspace_env (workspace_id, name, value) VALUES ($1, $2, $3) ON CONFLICT (workspace_id, name) DO UPDATE SET value = $3",
&w_id,
name,
value
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed.username,
"workspace.set_environment_variable",
ActionKind::Create,
&w_id,
Some(&authed.email),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Set environment variable {}", name))
}
None => {
sqlx::query!(
"DELETE FROM workspace_env WHERE workspace_id = $1 AND name = $2",
&w_id,
name
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed.username,
"workspace.delete_environment_variable",
ActionKind::Delete,
&w_id,
Some(&authed.email),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Deleted environment variable {}", name))
}
}
}
#[derive(Serialize)]
pub struct GetEncryptionKeyResponse {
key: String,
@@ -2960,6 +3026,14 @@ async fn change_workspace_id(
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE workspace_env SET workspace_id = $1 WHERE workspace_id = $2",
&rw.new_id,
&old_id
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE workspace_invite SET workspace_id = $1 WHERE workspace_id = $2",
&rw.new_id,
backend/windmill-common/src/variables.rs
+36 -7
View File
@@ -22,10 +22,10 @@ pub struct ContextualVariable {
pub name: String,
pub value: String,
pub description: String,
pub is_custom: bool,
}
#[derive(Serialize, Deserialize)]
#[derive(sqlx::FromRow)]
#[derive(Serialize, Deserialize, sqlx::FromRow)]
pub struct ListableVariable {
pub workspace_id: String,
@@ -42,8 +42,7 @@ pub struct ListableVariable {
pub is_linked: Option<bool>,
}
#[derive(Serialize, Deserialize)]
#[derive(sqlx::FromRow)]
#[derive(Serialize, Deserialize, sqlx::FromRow)]
pub struct ExportableListableVariable {
pub workspace_id: String,
@@ -133,6 +132,7 @@ pub async fn get_secret_value_as_admin(
}
pub async fn get_reserved_variables(
db: &DB,
w_id: &str,
token: &str,
email: &str,
@@ -146,7 +146,7 @@ pub async fn get_reserved_variables(
step_id: Option<String>,
root_flow_id: Option<String>,
jwt_token: Option<String>,
) -> [ContextualVariable; 17] {
) -> Vec<ContextualVariable> {
let state_path = {
let trigger = if schedule_path.is_some() {
username.to_string()
@@ -192,11 +192,12 @@ pub async fn get_reserved_variables(
format!("{joined_script_path}/{joined_schedule_path}/{ts}_{job_id}")
};
[
vec![
ContextualVariable {
name: "WM_WORKSPACE".to_string(),
value: w_id.to_string(),
description: "Workspace id of the current script".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_TOKEN".to_string(),
@@ -204,46 +205,55 @@ pub async fn get_reserved_variables(
description: "Token ephemeral to the current script with equal permission to the \
permission of the run (Usable as a bearer token)"
.to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_EMAIL".to_string(),
value: email.to_string(),
description: "Email of the user that executed the current script".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_USERNAME".to_string(),
value: username.to_string(),
description: "Username of the user that executed the current script".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_BASE_URL".to_string(),
value: BASE_URL.read().await.clone(),
description: "base url of this instance".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_JOB_ID".to_string(),
value: job_id.to_string(),
description: "Job id of the current script".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_JOB_PATH".to_string(),
value: path.unwrap_or_else(|| "".to_string()),
description: "Path of the script or flow being run if any".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_FLOW_JOB_ID".to_string(),
value: flow_id.unwrap_or_else(|| "".to_string()),
description: "Job id of the encapsulating flow if the job is a flow step".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_ROOT_FLOW_JOB_ID".to_string(),
value: root_flow_id.unwrap_or_else(|| "".to_string()),
description: "Job id of the root flow if the job is a flow step".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_FLOW_PATH".to_string(),
value: flow_path.unwrap_or_else(|| "".to_string()),
description: "Path of the encapsulating flow if the job is a flow step".to_string(),
is_custom: false,
},
ContextualVariable {
@@ -252,36 +262,55 @@ pub async fn get_reserved_variables(
description: "Path of the schedule if the job of the step or encapsulating step has \
been triggered by a schedule"
.to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_PERMISSIONED_AS".to_string(),
value: permissioned_as.to_string(),
description: "Fully Qualified (u/g) owner name of executor of the job".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_STATE_PATH".to_string(),
value: state_path.clone(),
description: "State resource path unique to a script and its trigger".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_STATE_PATH_NEW".to_string(),
value: state_path,
description: "State resource path unique to a script and its trigger (legacy)".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_FLOW_STEP_ID".to_string(),
value: step_id.unwrap_or_else(|| "".to_string()),
description: "The node id in a flow (like 'a', 'b', or 'f')".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_OBJECT_PATH".to_string(),
value: object_path,
description: "Script or flow step execution unique path, useful for storing results in an external service".to_string(),
is_custom: false,
},
ContextualVariable {
name: "WM_OIDC_JWT".to_string(),
value: jwt_token.unwrap_or_else(|| "".to_string()),
description: "OIDC JWT token (EE only)".to_string(),
is_custom: false,
},
]
].into_iter().chain( sqlx::query_as::<_, (String, String)>(
"SELECT name, value FROM workspace_env WHERE workspace_id = $1",
)
.bind(w_id)
.fetch_all(db)
.await
.unwrap_or_default()
.into_iter().map(|(name, value)| ContextualVariable {
name,
value,
description: "Custom workspace environment variable".to_string(),
is_custom: true,
})).collect()
}
backend/windmill-worker/src/bun_executor.rs
+1
View File
@@ -801,6 +801,7 @@ pub async fn start_worker(
get_common_bun_proc_envs(&base_internal_url).await;
let context = variables::get_reserved_variables(
db,
w_id,
&token,
"dedicated_worker@windmill.dev",
backend/windmill-worker/src/common.rs
+2
View File
@@ -284,6 +284,7 @@ pub async fn transform_json_value(
};
let variables = variables::get_reserved_variables(
db,
&job.workspace_id,
&client.token,
&job.email,
@@ -380,6 +381,7 @@ pub async fn get_reserved_variables(
};
let variables = variables::get_reserved_variables(
db,
&job.workspace_id,
token,
&job.email,
backend/windmill-worker/src/dedicated_worker.rs
+1 -1
View File
@@ -49,7 +49,7 @@ pub async fn handle_dedicated_process(
job_dir: &str,
context_envs: HashMap<String, String>,
envs: HashMap<String, String>,
reserved_variables: [variables::ContextualVariable; 17],
reserved_variables: Vec<variables::ContextualVariable>,
common_bun_proc_envs: HashMap<String, String>,
args: Vec<&str>,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
backend/windmill-worker/src/deno_executor.rs
+1
View File
@@ -421,6 +421,7 @@ pub async fn start_worker(
let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await;
let context = variables::get_reserved_variables(
db,
w_id,
&token,
"dedicated_worker@windmill.dev",
backend/windmill-worker/src/python_executor.rs
+2
View File
@@ -1046,6 +1046,7 @@ pub async fn start_worker(
let mut mem_peak: i32 = 0;
let mut canceled_by: Option<CanceledBy> = None;
let context = variables::get_reserved_variables(
db,
w_id,
&token,
"dedicated_worker@windmill.dev",
@@ -1163,6 +1164,7 @@ for line in sys.stdin:
}
let reserved_variables = windmill_common::variables::get_reserved_variables(
db,
w_id,
token,
"dedicated_worker",
frontend/src/lib/components/ContextualVariableEditor.svelte
+76
View File
@@ -0,0 +1,76 @@
<script lang="ts">
import { WorkspaceService } from '$lib/gen'
import { createEventDispatcher } from 'svelte'
import { workspaceStore } from '$lib/stores'
import { Button } from './common'
import Drawer from './common/drawer/Drawer.svelte'
import DrawerContent from './common/drawer/DrawerContent.svelte'
import autosize from 'svelte-autosize'
import { sendUserToast } from '$lib/toast'
import Section from './Section.svelte'
import { Save } from 'lucide-svelte'
const dispatch = createEventDispatcher()
let edit: boolean = false
let name: string = ''
let value: string = ''
export function initNew(): void {
edit = false
name = ''
value = ''
drawer.openDrawer()
}
export function editVariable(editName: string, editValue: string): void {
edit = true
name = editName
value = editValue
drawer.openDrawer()
}
let drawer: Drawer
async function updateVariable(): Promise<void> {
await WorkspaceService.setEnvironmentVariable({
workspace: $workspaceStore!,
requestBody: {
value: value,
name: name
}
})
sendUserToast(`${edit ? 'Updated' : 'Created'} contextual variable ${name}`)
dispatch('update')
drawer.closeDrawer()
}
</script>
<Drawer bind:this={drawer} size="900px">
<DrawerContent
title={edit ? `Update contextual variable ${name}` : 'Create a contextual variable'}
on:close={drawer.closeDrawer}
>
<div class="flex flex-col gap-8">
{#if !edit}
<Section label="Name">
<input type="text" bind:value={name} placeholder="Variable name" />
</Section>
{/if}
<Section label="Value">
<textarea rows="4" type="text" use:autosize bind:value placeholder="Variable value" />
</Section>
</div>
<svelte:fragment slot="actions">
<Button
on:click={() => updateVariable()}
disabled={value === '' || name === ''}
startIcon={{ icon: Save }}
color="dark"
size="sm"
>
{edit ? 'Update' : 'Save'}
</Button>
</svelte:fragment>
</DrawerContent>
</Drawer>
frontend/src/lib/components/TableSimple.svelte
+12
View File
@@ -1,4 +1,5 @@
<script lang="ts">
import DropdownV2 from './DropdownV2.svelte'
import Cell from './table/Cell.svelte'
import DataTable from './table/DataTable.svelte'
import Head from './table/Head.svelte'
@@ -7,6 +8,8 @@
export let data: any[] | undefined // Object containing the data
export let keys: string[]
export let size: 'sm' | 'md' | 'lg' = 'md'
export let getRowActions: ((row: any) => any[]) | undefined = undefined
</script>
<div class="mt-2 w-full">
@@ -18,12 +21,16 @@
<Cell first={i == 0} last={i == headers.length - 1} head class="max-w-96">{header}</Cell
>
{/each}
{#if getRowActions !== undefined}
<Cell head last />
{/if}
{/if}
</tr>
</Head>
<tbody class="divide-y">
{#if data && keys && data.length > 0}
{#each data as row}
{@const rowActions = getRowActions?.(row)}
<tr>
{#each keys as key, i}
<Cell
@@ -34,6 +41,11 @@
{row[key] ?? ''}
</Cell>
{/each}
{#if rowActions && rowActions.length > 0}
<Cell last shouldStopPropagation>
<DropdownV2 items={rowActions} />
</Cell>
{/if}
</tr>
{/each}
{:else}
frontend/src/routes/(root)/(logged)/variables/+page.svelte
+63 -5
View File
@@ -2,6 +2,7 @@
import CenteredPage from '$lib/components/CenteredPage.svelte'
import { Alert, Badge, Button, Skeleton, Tab, Tabs } from '$lib/components/common'
import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte'
import ContextualVariableEditor from '$lib/components/ContextualVariableEditor.svelte'
import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte'
import Dropdown from '$lib/components/DropdownV2.svelte'
import ListFilters from '$lib/components/home/ListFilters.svelte'
@@ -18,7 +19,7 @@
import Tooltip from '$lib/components/Tooltip.svelte'
import VariableEditor from '$lib/components/VariableEditor.svelte'
import type { ContextualVariable, ListableVariable } from '$lib/gen'
import { OauthService, VariableService } from '$lib/gen'
import { OauthService, VariableService, WorkspaceService } from '$lib/gen'
import { userStore, workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { canWrite, isOwner, truncate } from '$lib/utils'
@@ -44,6 +45,7 @@
let contextualVariables: ContextualVariable[] = []
let shareModal: ShareModal
let variableEditor: VariableEditor
let contextualVariableEditor: ContextualVariableEditor
let loading = {
contextual: true
}
@@ -101,6 +103,18 @@
let tab: 'workspace' | 'contextual' = 'workspace'
let deploymentDrawer: DeployWorkspaceDrawer
async function deleteContextualVariable(row: { name: string }) {
await WorkspaceService.setEnvironmentVariable({
workspace: $workspaceStore!,
requestBody: {
name: row.name,
value: undefined
}
})
loadContextualVariables()
sendUserToast(`Custom contextual variable ${row.name} was deleted`)
}
</script>
<DeployWorkspaceDrawer bind:this={deploymentDrawer} />
@@ -119,13 +133,27 @@
documentationLink="https://www.windmill.dev/docs/core_concepts/variables_and_secrets"
>
<div class="flex flex-row justify-end">
<Button size="md" startIcon={{ icon: Plus }} on:click={() => variableEditor.initNew()}>
New&nbsp;variable
</Button>
{#if tab == 'contextual' && $userStore?.is_admin}
<Button
size="md"
startIcon={{ icon: Plus }}
on:click={() => contextualVariableEditor.initNew()}
>
New&nbsp;contextual&nbsp;variable
</Button>
{:else}
<Button size="md" startIcon={{ icon: Plus }} on:click={() => variableEditor.initNew()}>
New&nbsp;variable
</Button>
{/if}
</div>
</PageHeader>
<VariableEditor bind:this={variableEditor} on:create={loadVariables} />
<ContextualVariableEditor
bind:this={contextualVariableEditor}
on:update={loadContextualVariables}
/>
<ShareModal
bind:this={shareModal}
on:change={() => {
@@ -370,9 +398,39 @@
<Skeleton layout={[[2.8], 0.5]} />
{/each}
{:else}
<PageHeader title="Custom contextual variables" primary={false} />
{#if contextualVariables.filter((x) => x.is_custom).length === 0}
<div class="flex flex-col items-center justify-center h-full">
<div class="text-md font-medium">No custom contextual variables found</div>
</div>
{:else}
<TableSimple
headers={['Name', 'Value']}
data={contextualVariables.filter((x) => x.is_custom)}
keys={['name', 'value']}
getRowActions={$userStore?.is_admin
? (row) => {
return [
{
displayName: 'Edit',
action: () => contextualVariableEditor.editVariable(row.name, row.value)
},
{
displayName: 'Delete',
type: 'delete',
action: () => {
deleteContextualVariable(row)
}
}
]
}
: undefined}
/>
{/if}
<PageHeader title="Contextual variables" primary={false} />
<TableSimple
headers={['Name', 'Example of value', 'Description']}
data={contextualVariables}
data={contextualVariables.filter((x) => !x.is_custom)}
keys={['name', 'value', 'description']}
/>
{/if}