From 4f43b1984f4ea9a87b0489d4b40c1ecdcfdbdecd Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 14 Mar 2025 15:17:48 +0100 Subject: [PATCH] feat: keep captures across drafts and deploys (#5482) --- ...39319a854f761a300970e2fd909f8058ec566.json | 17 ++++++++ ...6874ee8b277a2914dff6f9c33a1f5a2351114.json | 17 ++++++++ backend/windmill-api/openapi.yaml | 39 +++++++++++++++++++ backend/windmill-api/src/capture.rs | 39 +++++++++++++++++++ backend/windmill-api/src/flows.rs | 36 ++++++++++------- backend/windmill-api/src/scripts.rs | 36 ++++++++++------- frontend/src/lib/components/Dev.svelte | 3 +- .../src/lib/components/FlowBuilder.svelte | 32 +++++++++++++-- .../lib/components/FlowPreviewContent.svelte | 9 +++-- .../src/lib/components/ScriptBuilder.svelte | 27 ++++++++++++- .../flows/content/FlowEditorPanel.svelte | 6 ++- .../components/flows/content/FlowInput.svelte | 6 +-- .../flows/content/FlowSettings.svelte | 6 +-- frontend/src/lib/components/flows/types.ts | 3 +- .../components/triggers/TriggersEditor.svelte | 19 ++++----- frontend/src/routes/flows/dev/+page.svelte | 3 +- 16 files changed, 241 insertions(+), 57 deletions(-) create mode 100644 backend/.sqlx/query-7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566.json create mode 100644 backend/.sqlx/query-e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114.json diff --git a/backend/.sqlx/query-7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566.json b/backend/.sqlx/query-7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566.json new file mode 100644 index 0000000000..4e6d8e9ba2 --- /dev/null +++ b/backend/.sqlx/query-7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "7e891e053b8545c800d629421f239319a854f761a300970e2fd909f8058ec566" +} diff --git a/backend/.sqlx/query-e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114.json b/backend/.sqlx/query-e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114.json new file mode 100644 index 0000000000..461721c4c2 --- /dev/null +++ b/backend/.sqlx/query-e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "e70c3da5864b7946735b7ad6c416874ee8b277a2914dff6f9c33a1f5a2351114" +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4716daeb12..2b4bd94f9c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4668,6 +4668,11 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - name: keep_captures + description: keep captures + in: query + schema: + type: boolean responses: "200": description: script path @@ -5613,6 +5618,11 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/ScriptPath" + - name: keep_captures + description: keep captures + in: query + schema: + type: boolean responses: "200": description: flow delete @@ -10602,6 +10612,35 @@ paths: type: array items: $ref: "#/components/schemas/Capture" + + + /w/{workspace}/capture/move/{runnable_kind}/{path}: + post: + summary: move captures and configs for a script or flow + operationId: moveCapturesAndConfigs + tags: + - capture + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/RunnableKind" + - $ref: "#/components/parameters/Path" + requestBody: + description: move captures and configs to a new path + required: true + content: + application/json: + schema: + type: object + properties: + new_path: + type: string + responses: + "200": + description: captures and configs moved + content: + text/plain: + schema: + type: string /w/{workspace}/capture/{id}: get: diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index f52cee3fab..402d504a2c 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -67,6 +67,10 @@ pub fn workspaced_service() -> Router { ) .route("/get_configs/:runnable_kind/*path", get(get_configs)) .route("/list/:runnable_kind/*path", get(list_captures)) + .route( + "/move/:runnable_kind/*path", + post(move_captures_and_configs), + ) .route("/:id", delete(delete_capture)) .route("/:id", get(get_capture)) } @@ -450,6 +454,41 @@ async fn delete_capture( Ok(()) } +#[derive(Deserialize)] +struct MoveCapturesAndConfigsBody { + new_path: String, +} + +async fn move_captures_and_configs( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, runnable_kind, old_path)): Path<(String, RunnableKind, StripPath)>, + Json(body): Json, +) -> Result<()> { + let mut tx = user_db.begin(&authed).await?; + let old_path = old_path.to_path(); + sqlx::query!( + "UPDATE capture_config SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4", + body.new_path, + old_path, + &w_id, + matches!(runnable_kind, RunnableKind::Flow), + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "UPDATE capture SET path = $1 WHERE path = $2 AND workspace_id = $3 AND is_flow = $4", + body.new_path, + old_path, + &w_id, + matches!(runnable_kind, RunnableKind::Flow), + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + #[derive(Serialize, Deserialize)] struct ActiveCaptureOwner { owner: String, diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 8cd65dd55a..beaec364f6 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1183,12 +1183,18 @@ async fn archive_flow_by_path( Ok(format!("Flow {path} archived")) } +#[derive(Deserialize)] +struct DeleteFlowQuery { + keep_captures: Option, +} + async fn delete_flow_by_path( authed: ApiAuthed, Extension(db): Extension, Extension(user_db): Extension, Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, ) -> Result { let path = path.to_path(); let mut tx = user_db.begin(&authed).await?; @@ -1209,21 +1215,23 @@ async fn delete_flow_by_path( .execute(&mut *tx) .await?; - sqlx::query!( - "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", - path, - &w_id - ) - .execute(&mut *tx) - .await?; + if !query.keep_captures.unwrap_or(false) { + sqlx::query!( + "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", + path, + &w_id + ) + .execute(&mut *tx) + .await?; - sqlx::query!( - "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", - path, - &w_id - ) - .execute(&mut *tx) - .await?; + sqlx::query!( + "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS TRUE", + path, + &w_id + ) + .execute(&mut *tx) + .await?; + } audit_log( &mut *tx, diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index d1c6e43902..e72ec6f167 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -1481,12 +1481,18 @@ async fn delete_script_by_hash( Ok(Json(script)) } +#[derive(Deserialize)] +struct DeleteScriptQuery { + keep_captures: Option, +} + async fn delete_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, Extension(webhook): Extension, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, ) -> JsonResult { let path = path.to_path(); @@ -1540,21 +1546,23 @@ async fn delete_script_by_path( .execute(&db) .await?; - sqlx::query!( - "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", - path, - w_id - ) - .execute(&db) - .await?; + if !query.keep_captures.unwrap_or(false) { + sqlx::query!( + "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", + path, + w_id + ) + .execute(&db) + .await?; - sqlx::query!( - "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", - path, - w_id - ) - .execute(&db) - .await?; + sqlx::query!( + "DELETE FROM capture WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", + path, + w_id + ) + .execute(&db) + .await?; + } audit_log( &mut *tx, diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 84b0d3049a..8e96e68e7c 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -528,7 +528,8 @@ flowStore, testStepStore, saveDraft: () => {}, - initialPath: '', + initialPathStore: writable(''), + fakeInitialPath: '', flowInputsStore: writable({}), customUi: {}, insertButtonOpen: writable(false), diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index ca43689495..d09f34ce7d 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -10,7 +10,8 @@ type OpenFlow, type RawScript, type InputTransform, - type TriggersCount + type TriggersCount, + CaptureService } from '$lib/gen' import { initHistory, push, redo, undo } from '$lib/history' import { @@ -24,6 +25,7 @@ cleanValueProperties, encodeState, formatCron, + generateRandomString, orderedJsonStringify, replaceFalseWithUndefined, sleep, @@ -105,6 +107,18 @@ export let version: number | undefined = undefined export let setSavedraftCb: ((cb: () => void) => void) | undefined = undefined + let initialPathStore = writable(initialPath) + $: initialPathStore.set(initialPath) + + // used for new flows for captures + let fakeInitialPath = + 'u/' + + ($userStore?.username?.includes('@') + ? $userStore!.username.split('@')[0].replace(/[^a-zA-Z0-9_]/g, '') + : $userStore!.username!) + + '/' + + generateRandomString(12) + // Used by multiplayer deploy collision warning let deployedValue: Value | undefined = undefined // Value to diff against let deployedBy: string | undefined = undefined // Author @@ -216,7 +230,18 @@ if (savedFlow?.draft_only) { await FlowService.deleteFlowByPath({ workspace: $workspaceStore!, - path: initialPath + path: initialPath, + keepCaptures: true + }) + } + if (!initialPath || $pathStore != initialPath) { + await CaptureService.moveCapturesAndConfigs({ + workspace: $workspaceStore!, + path: initialPath || fakeInitialPath, + requestBody: { + new_path: $pathStore + }, + runnableKind: 'flow' }) } await FlowService.createFlow({ @@ -544,7 +569,8 @@ pathStore, testStepStore, saveDraft, - initialPath, + initialPathStore, + fakeInitialPath, flowInputsStore: writable({}), customUi, insertButtonOpen, diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index d541a46397..e9cba15af9 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -51,7 +51,7 @@ flowStateStore, flowStore, pathStore, - initialPath, + initialPathStore, customUi, executionCount } = getContext('FlowEditorContext') @@ -182,7 +182,8 @@ } const previousJobId = await JobService.listJobs({ workspace: $workspaceStore!, - scriptPathExact: (initialPath == '' ? $pathStore : initialPath) + '/' + module.id, + scriptPathExact: + ($initialPathStore == '' ? $pathStore : $initialPathStore) + '/' + module.id, jobKinds: ['preview', 'script', 'flowpreview', 'flow'].join(','), page: 1, perPage: 1 @@ -372,7 +373,7 @@
{#if jobId} diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index d112455e72..4fc5481ac8 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -7,7 +7,8 @@ ScheduleService, type Script, type TriggersCount, - PostgresTriggerService + PostgresTriggerService, + CaptureService } from '$lib/gen' import { inferArgs } from '$lib/infer' import { initialCode } from '$lib/script_helpers' @@ -18,6 +19,7 @@ emptyString, encodeState, formatCron, + generateRandomString, orderedJsonStringify, replaceFalseWithUndefined, type Value @@ -106,6 +108,15 @@ } } + // used for new scripts for captures + let fakeInitialPath = + 'u/' + + ($userStore?.username?.includes('@') + ? $userStore!.username.split('@')[0].replace(/[^a-zA-Z0-9_]/g, '') + : $userStore!.username!) + + '/' + + generateRandomString(12) + let deployedValue: Value | undefined = undefined // Value to diff against let deployedBy: string | undefined = undefined // Author let confirmCallback: () => void = () => {} // What happens when user clicks `override` in warning @@ -565,10 +576,21 @@ if (savedScript?.draft_only) { await ScriptService.deleteScriptByPath({ workspace: $workspaceStore!, - path: initialPath + path: initialPath, + keepCaptures: true }) script.parent_hash = undefined } + if (!initialPath || script.path != initialPath) { + await CaptureService.moveCapturesAndConfigs({ + workspace: $workspaceStore!, + path: initialPath || fakeInitialPath, + requestBody: { + new_path: script.path + }, + runnableKind: 'script' + }) + } await ScriptService.createScript({ workspace: $workspaceStore!, requestBody: { @@ -1394,6 +1416,7 @@ }} args={hasPreprocessor && selectedInputTab !== 'preprocessor' ? {} : args} {initialPath} + {fakeInitialPath} schema={script.schema} noEditor={true} isFlow={false} diff --git a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte index 77be332ad5..dd710eac9e 100644 --- a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte @@ -25,7 +25,8 @@ flowStateStore, flowInputsStore, pathStore, - initialPath, + initialPathStore, + fakeInitialPath, previewArgs, flowInputEditorState } = getContext('FlowEditorContext') @@ -114,7 +115,8 @@ on:testWithArgs args={$previewArgs} currentPath={$pathStore} - {initialPath} + initialPath={$initialPathStore} + {fakeInitialPath} schema={$flowStore.schema} {noEditor} newItem={newFlow} diff --git a/frontend/src/lib/components/flows/content/FlowInput.svelte b/frontend/src/lib/components/flows/content/FlowInput.svelte index 3466551844..7748bd9613 100644 --- a/frontend/src/lib/components/flows/content/FlowInput.svelte +++ b/frontend/src/lib/components/flows/content/FlowInput.svelte @@ -47,7 +47,7 @@ export let noEditor: boolean export let disabled: boolean - const { flowStore, previewArgs, pathStore, initialPath, flowInputEditorState } = + const { flowStore, previewArgs, pathStore, initialPathStore, flowInputEditorState } = getContext('FlowEditorContext') let addProperty: AddPropertyV2 | undefined = undefined @@ -483,7 +483,7 @@ > { updatePreviewSchemaAndArgs(e.detail?.args ?? undefined) @@ -525,7 +525,7 @@ title="Saved inputs" > { updatePreviewSchemaAndArgs(e.detail ?? undefined) diff --git a/frontend/src/lib/components/flows/content/FlowSettings.svelte b/frontend/src/lib/components/flows/content/FlowSettings.svelte index 4737d0446b..2c46210a1e 100644 --- a/frontend/src/lib/components/flows/content/FlowSettings.svelte +++ b/frontend/src/lib/components/flows/content/FlowSettings.svelte @@ -21,7 +21,7 @@ export let noEditor: boolean - const { flowStore, initialPath, previewArgs, pathStore, customUi } = + const { flowStore, initialPathStore, previewArgs, pathStore, customUi } = getContext('FlowEditorContext') function asSchema(x: any) { @@ -67,7 +67,7 @@ promptConfigName="flowSummary" flow={$flowStore.value} on:change={() => { - if (initialPath == '' && $flowStore.summary?.length > 0 && !dirtyPath) { + if ($initialPathStore == '' && $flowStore.summary?.length > 0 && !dirtyPath) { path?.setName( $flowStore.summary .toLowerCase() @@ -92,7 +92,7 @@ bind:this={path} bind:dirty={dirtyPath} bind:path={$pathStore} - {initialPath} + initialPath={$initialPathStore} namePlaceholder="flow" kind="flow" /> diff --git a/frontend/src/lib/components/flows/types.ts b/frontend/src/lib/components/flows/types.ts index 76f8064ffb..b72c321615 100644 --- a/frontend/src/lib/components/flows/types.ts +++ b/frontend/src/lib/components/flows/types.ts @@ -51,7 +51,8 @@ export type FlowEditorContext = { flowStateStore: Writable testStepStore: Writable> saveDraft: () => void - initialPath: string + initialPathStore: Writable + fakeInitialPath: string flowInputsStore: Writable customUi: FlowBuilderWhitelabelCustomUi insertButtonOpen: Writable diff --git a/frontend/src/lib/components/triggers/TriggersEditor.svelte b/frontend/src/lib/components/triggers/TriggersEditor.svelte index ce5a4f24ce..9e3bcb9db9 100644 --- a/frontend/src/lib/components/triggers/TriggersEditor.svelte +++ b/frontend/src/lib/components/triggers/TriggersEditor.svelte @@ -24,6 +24,7 @@ export let noEditor: boolean export let newItem = false export let currentPath: string + export let fakeInitialPath: string export let hash: string | undefined = undefined export let initialPath: string export let schema: any @@ -85,7 +86,7 @@ on:updateSchema on:testWithArgs scopes={isFlow ? [`run:flow/${currentPath}`] : [`run:script/${currentPath}`]} - path={currentPath} + path={initialPath || fakeInitialPath} {hash} {isFlow} {args} @@ -105,7 +106,7 @@ on:testWithArgs token="" scopes={isFlow ? [`run:flow/${currentPath}`] : [`run:script/${currentPath}`]} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} @@ -122,7 +123,7 @@ on:testWithArgs {newItem} {args} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} @@ -137,7 +138,7 @@ on:updateSchema on:testWithArgs {newItem} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} @@ -152,7 +153,7 @@ on:updateSchema on:testWithArgs {newItem} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} {canHavePreprocessor} {hasPreprocessor} @@ -174,7 +175,7 @@ on:updateSchema on:testWithArgs {newItem} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} @@ -185,7 +186,7 @@ on:applyArgs on:addPreprocessor {newItem} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} @@ -198,7 +199,7 @@ on:updateSchema on:testWithArgs {newItem} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} @@ -211,7 +212,7 @@ on:updateSchema on:testWithArgs {newItem} - path={currentPath} + path={initialPath || fakeInitialPath} {isFlow} isEditor={true} {canHavePreprocessor} diff --git a/frontend/src/routes/flows/dev/+page.svelte b/frontend/src/routes/flows/dev/+page.svelte index db36f81d93..cffef6240c 100644 --- a/frontend/src/routes/flows/dev/+page.svelte +++ b/frontend/src/routes/flows/dev/+page.svelte @@ -103,7 +103,8 @@ flowStore, testStepStore, saveDraft: () => {}, - initialPath: '', + initialPathStore: writable(''), + fakeInitialPath: '', flowInputsStore: writable({}), customUi: {}, insertButtonOpen: writable(false),