From 9970f6daeba1e70636f6ac51decfcb82af1885e0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Aug 2023 18:54:40 +0200 Subject: [PATCH] feat: migrate state path to new schema --- .../20230811145254_step_id_to_queue.down.sql | 1 + .../20230811145254_step_id_to_queue.up.sql | 2 + backend/windmill-api/src/apps.rs | 1 + backend/windmill-api/src/flows.rs | 2 + backend/windmill-api/src/jobs.rs | 11 +++++ backend/windmill-api/src/oauth2.rs | 1 + backend/windmill-api/src/scripts.rs | 1 + backend/windmill-api/src/variables.rs | 1 + backend/windmill-common/src/jobs.rs | 3 ++ backend/windmill-common/src/variables.rs | 39 ++++++++++++++- backend/windmill-queue/src/jobs.rs | 8 +++- backend/windmill-queue/src/schedule.rs | 1 + backend/windmill-worker/src/worker.rs | 3 +- backend/windmill-worker/src/worker_flow.rs | 1 + deno-client/mod.ts | 2 +- .../src/lib/components/ModulePreview.svelte | 2 +- .../apps/editor/AppEditorHeader.svelte | 48 ++++++++++++++----- .../InlineScriptEditor.svelte | 6 ++- .../src/lib/components/runs/RunsTable.svelte | 1 - go-client/windmill.go | 2 +- python-client/wmill/wmill/client.py | 2 +- typescript-client/client.ts | 2 +- 22 files changed, 116 insertions(+), 24 deletions(-) create mode 100644 backend/migrations/20230811145254_step_id_to_queue.down.sql create mode 100644 backend/migrations/20230811145254_step_id_to_queue.up.sql diff --git a/backend/migrations/20230811145254_step_id_to_queue.down.sql b/backend/migrations/20230811145254_step_id_to_queue.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20230811145254_step_id_to_queue.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20230811145254_step_id_to_queue.up.sql b/backend/migrations/20230811145254_step_id_to_queue.up.sql new file mode 100644 index 0000000000..2437da5d26 --- /dev/null +++ b/backend/migrations/20230811145254_step_id_to_queue.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE queue ADD COLUMN flow_step_id VARCHAR(255); \ No newline at end of file diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 16c1a28d01..93397774df 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -809,6 +809,7 @@ async fn execute_component( true, tag, None, + None, ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 42ada12f46..1da14638a0 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -265,6 +265,7 @@ async fn create_flow( true, nf.tag, None, + None, ) .await?; @@ -448,6 +449,7 @@ async fn update_flow( true, None, None, + None, ) .await?; sqlx::query!( diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 21bdd86e3c..f7b736ade4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -1241,6 +1241,7 @@ impl From for Job { concurrent_limit: uj.concurrent_limit, concurrency_time_window_s: uj.concurrency_time_window_s, timeout: None, + flow_step_id: None, }), t => panic!("job type {} not valid", t), } @@ -1476,6 +1477,7 @@ pub async fn run_flow_by_path( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; @@ -1521,6 +1523,7 @@ pub async fn run_job_by_path( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; @@ -1723,6 +1726,7 @@ pub async fn run_wait_result_job_by_path_get( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; @@ -1893,6 +1897,7 @@ async fn run_wait_result_script_by_path_internal( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; @@ -1953,6 +1958,7 @@ pub async fn run_wait_result_script_by_hash( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; @@ -2058,6 +2064,7 @@ async fn run_wait_result_flow_by_path_internal( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; @@ -2122,6 +2129,7 @@ async fn run_preview_job( true, preview.tag, None, + None, ) .await?; tx.commit().await?; @@ -2159,6 +2167,7 @@ async fn add_noop_jobs( true, None, None, + None, ) .await?; tx = ntx; @@ -2207,6 +2216,7 @@ async fn run_preview_flow_job( true, raw_flow.tag, None, + None, ) .await?; tx.commit().await?; @@ -2258,6 +2268,7 @@ pub async fn run_job_by_hash( !run_query.invisible_to_owner.unwrap_or(false), tag, None, + None, ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/oauth2.rs b/backend/windmill-api/src/oauth2.rs index ccab759f14..06c41ff9ed 100644 --- a/backend/windmill-api/src/oauth2.rs +++ b/backend/windmill-api/src/oauth2.rs @@ -841,6 +841,7 @@ async fn slack_command( true, tag, None, + None, ) .await?; let url = BASE_URL.to_owned(); diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 5b2fad15a0..634743368a 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -513,6 +513,7 @@ async fn create_script( true, ns.tag, None, + None, ) .await?; tx = new_tx; diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index a8f488614c..3a5a7eb8b5 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -64,6 +64,7 @@ async fn list_contextual_variables( Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), Some("u/user/encapsulating_flow_path".to_string()), Some("u/user/triggering_flow_path".to_string()), + Some("c".to_string()), ) .to_vec(), )) diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index c45093f40b..b5424ba4ff 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -87,6 +87,8 @@ pub struct QueuedJob { pub concurrency_time_window_s: Option, #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_step_id: Option, } impl QueuedJob { @@ -152,6 +154,7 @@ impl Default for QueuedJob { concurrent_limit: None, concurrency_time_window_s: None, timeout: None, + flow_step_id: None, } } } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 3262c05199..fc09ec1eb4 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -72,7 +72,8 @@ pub fn get_reserved_variables( flow_id: Option, flow_path: Option, schedule_path: Option, -) -> [ContextualVariable; 12] { + step_id: Option, +) -> [ContextualVariable; 14] { let state_path = { let flow_path = flow_path .clone() @@ -88,8 +89,34 @@ pub fn get_reserved_variables( } else { script_path }; + format!("{permissioned_as}/{flow_path}/{script_path}{schedule_path}") }; + + let state_path_2 = { + let trigger = if schedule_path.is_some() { + username.to_string() + } else { + "user".to_string() + }; + + if let Some(flow_path) = flow_path.clone() { + format!( + "{flow_path}/{}_{trigger}", + step_id.clone().unwrap_or_else(|| "nostep".to_string()) + ) + } else if let Some(script_path) = path.clone() { + let script_path = if script_path.ends_with("/") { + "noname".to_string() + } else { + script_path + }; + format!("{script_path}/{trigger}") + } else { + format!("u/{username}/tmp_state") + } + }; + [ ContextualVariable { name: "WM_WORKSPACE".to_string(), @@ -153,7 +180,17 @@ pub fn get_reserved_variables( ContextualVariable { name: "WM_STATE_PATH".to_string(), value: state_path, + description: "State resource path unique to a script and its trigger (legacy, in a migration period against WM_STATE_PATH_NEW)".to_string(), + }, + ContextualVariable { + name: "WM_STATE_PATH_NEW".to_string(), + value: state_path_2, description: "State resource path unique to a script and its trigger".to_string(), }, + ContextualVariable { + name: "WM_FLOW_STEP_ID".to_string(), + value: step_id.unwrap_or_else(|| "".to_string()), + description: "The node id in a flow (like 'a', 'b', or 'f')".to_string(), + }, ] } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index dec01c0b9b..7e7e2c4c37 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -476,6 +476,7 @@ pub async fn run_error_handler( true, tag, None, + None, ) .await?; tx.commit().await?; @@ -677,6 +678,7 @@ async fn handle_on_failure<'c, R: rsmq_async::RsmqConnection + Clone + Send + 'c true, tag, None, + None, ) .await?; tracing::info!( @@ -1022,6 +1024,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( visible_to_owner: bool, mut tag: Option, custom_timeout: Option, + flow_step_id: Option, ) -> Result<(Uuid, QueueTransaction<'c, R>), Error> { let args_json = serde_json::Value::Object(args); let job_id: Uuid = if let Some(job_id) = job_id { @@ -1350,8 +1353,8 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( "INSERT INTO queue (workspace_id, id, running, parent_job, created_by, permissioned_as, scheduled_for, script_hash, script_path, raw_code, raw_lock, args, job_kind, schedule_path, raw_flow, \ - flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag, concurrent_limit, concurrency_time_window_s, timeout) - VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24, $25, $26, $27) \ + flow_status, is_flow_step, language, started_at, same_worker, pre_run_error, email, visible_to_owner, root_job, tag, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id) + VALUES ($1, $2, $3, $4, $5, $6, COALESCE($7, now()), $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, CASE WHEN $3 THEN now() END, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28) \ RETURNING id", workspace_id, job_id, @@ -1380,6 +1383,7 @@ pub async fn push<'c, R: rsmq_async::RsmqConnection + Send + 'c>( concurrent_limit, concurrency_time_window_s, custom_timeout, + flow_step_id ) .fetch_one(&mut tx) .await diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 3d356037d3..61af38a9ad 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -116,6 +116,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( true, tag, None, + None, ) .await?; Ok(tx) // TODO: Bubble up pushed UUID from here diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 575927a818..53895963ff 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2385,7 +2385,8 @@ pub async fn get_reserved_variables( job.script_path.clone(), job.parent_job.map(|x| x.to_string()), flow_path, - job.schedule_path.clone() + job.schedule_path.clone(), + job.flow_step_id.clone() ).to_vec(); let mut r: HashMap = variables diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index aec7248557..b453a265ce 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1433,6 +1433,7 @@ async fn push_next_flow_job Some(flow_job.tag.clone()) }, module.timeout, + Some(module.id.clone()), ) .await?; tx = inner_tx; diff --git a/deno-client/mod.ts b/deno-client/mod.ts index 8361c64589..fafef0cc59 100644 --- a/deno-client/mod.ts +++ b/deno-client/mod.ts @@ -112,7 +112,7 @@ export async function getFullResource( } export function getStatePath(): string { - const state_path = Deno.env.get("WM_STATE_PATH"); + const state_path = Deno.env.get("WM_STATE_PATH_NEW"); if (state_path === undefined) { throw Error("State path not set"); } diff --git a/frontend/src/lib/components/ModulePreview.svelte b/frontend/src/lib/components/ModulePreview.svelte index 9f376668b9..89f91170b4 100644 --- a/frontend/src/lib/components/ModulePreview.svelte +++ b/frontend/src/lib/components/ModulePreview.svelte @@ -54,7 +54,7 @@ // let jobId: string | undefined = undefined if (val.type == 'rawscript') { await testJobLoader?.runPreview( - val.path, + val.path ?? ($flowStore?.path ?? '') + '/' + mod.id, val.content, val.language, args, diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte index 9a6b338344..4a3683fb77 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeader.svelte @@ -387,18 +387,8 @@ lock = false } + let dirtyPath = false let path: Path | undefined = undefined - $: { - if (appPath == '' && $summary?.length > 0) { - path?.setName( - $summary - .toLowerCase() - .replace(/[^a-z0-9_]/g, '_') - .replace(/-+/g, '_') - .replace(/^-|-$/g, '') - ) - } - } @@ -416,18 +406,31 @@

Summary

{ + if (appPath == '' && $summary?.length > 0 && !dirtyPath) { + path?.setName( + $summary + .toLowerCase() + .replace(/[^a-z0-9_]/g, '_') + .replace(/-+/g, '_') + .replace(/^-|-$/g, '') + ) + } + }} />
-
closeSaveDrawer()}> Summary
- + { + if (appPath == '' && $summary?.length > 0 && !dirtyPath) { + path?.setName( + $summary + .toLowerCase() + .replace(/[^a-z0-9_]/g, '_') + .replace(/-+/g, '_') + .replace(/^-|-$/g, '') + ) + } + }} + />
Path
diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte index 8d83785586..5b0d3cdece 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte @@ -56,7 +56,11 @@ return schema } - $: inlineScript && (inlineScript.path = `${defaultIfEmptyString(appPath, '')}/${name}`) + $: inlineScript && + (inlineScript.path = `${defaultIfEmptyString(appPath, 'new_app')}/${name?.replaceAll( + ' ', + '_' + )}`) onMount(async () => { if (inlineScript && !inlineScript.schema) { diff --git a/frontend/src/lib/components/runs/RunsTable.svelte b/frontend/src/lib/components/runs/RunsTable.svelte index c9c1142051..386ed49c12 100644 --- a/frontend/src/lib/components/runs/RunsTable.svelte +++ b/frontend/src/lib/components/runs/RunsTable.svelte @@ -58,7 +58,6 @@ console.log('load more')} loadMore={loadMoreQuantity} shouldLoadMore={nbObJobs < jobs.length} on:loadMore={() => (nbObJobs += loadMoreQuantity)} diff --git a/go-client/windmill.go b/go-client/windmill.go index 8f8f3d2aa8..26832c060e 100644 --- a/go-client/windmill.go +++ b/go-client/windmill.go @@ -103,7 +103,7 @@ func SetVariable(path string, value string) error { } func GetStatePath() string { - return os.Getenv("WM_STATE_PATH") + return os.Getenv("WM_STATE_PATH_NEW") } func GetState() (interface{}, error) { diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index ccbe183a89..df735f93f9 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -334,7 +334,7 @@ def set_variable(path: str, value: str) -> None: def get_state_path() -> str: - state_path = os.environ.get("WM_STATE_PATH") + state_path = os.environ.get("WM_STATE_PATH_NEW") if state_path is None: raise Exception("State path not found") return state_path diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 6977de6016..b64713a802 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -119,7 +119,7 @@ export async function getFullResource( } export function getStatePath(): string { - const state_path = getEnv("WM_STATE_PATH"); + const state_path = getEnv("WM_STATE_PATH_NEW"); if (state_path === undefined) { throw Error("State path not set"); }