feat: Add possibility to delete flow step results when the flow is complete (#2806)

* feat: Add possibility to delete flow step results when the flow is complete

* Add third layer of tabs and gate feature to EE

frontend/src/lib/components/ScriptBuilder.svelte

@@ -214,7 +214,8 @@
 					cache_ttl: script.cache_ttl,
 					ws_error_handler_muted: script.ws_error_handler_muted,
 					priority: script.priority,
-					restart_unless_cancelled: script.restart_unless_cancelled
+					restart_unless_cancelled: script.restart_unless_cancelled,
+					delete_after_use: script.delete_after_use
 				}
 			})
 			savedScript = cloneDeep(script) as NewScriptWithDraft
@@ -289,7 +290,8 @@
 						cache_ttl: script.cache_ttl,
 						ws_error_handler_muted: script.ws_error_handler_muted,
 						priority: script.priority,
-						restart_unless_cancelled: script.restart_unless_cancelled
+						restart_unless_cancelled: script.restart_unless_cancelled,
+						delete_after_use: script.delete_after_use
 					}
 				})
 			}
@@ -707,6 +709,44 @@
 										>
 									</svelte:fragment>
 								</Section>
+								<Section label="Delete after use">
+									<svelte:fragment slot="header">
+										<Tooltip>
+											WARNING: This settings ONLY applies to synchronous webhooks or when the script
+											is used within a flow. If used individually, this script must be triggered
+											using a synchronous endpoint to have the desired effect.
+											<br />
+											<br />
+											The logs, arguments and results of the job will be completely deleted from Windmill
+											once it is complete and the result has been returned.
+											<br />
+											<br />
+											The deletion is irreversible.
+											{#if !$enterpriseLicense}
+												<br />
+												<br />
+												This option is only available on Windmill Enterprise Edition.
+											{/if}
+										</Tooltip>
+									</svelte:fragment>
+									<div class="flex gap-2 shrink flex-col">
+										<Toggle
+											disabled={!$enterpriseLicense}
+											size="sm"
+											checked={Boolean(script.delete_after_use)}
+											on:change={() => {
+												if (script.delete_after_use) {
+													script.delete_after_use = undefined
+												} else {
+													script.delete_after_use = true
+												}
+											}}
+											options={{
+												right: 'Delete logs, arguments and results after use'
+											}}
+										/>
+									</div>
+								</Section>
 								{#if !isCloudHosted()}
 									<Section label="High priority script" eeOnly>
 										<Toggle

frontend/src/lib/components/flows/content/FlowBranchesAllWrapper.svelte

@@ -12,6 +12,8 @@
 	import FlowModuleSleep from './FlowModuleSleep.svelte'
 	import FlowModuleSuspend from './FlowModuleSuspend.svelte'
 	import FlowModuleMock from './FlowModuleMock.svelte'
+	import FlowModuleDeleteAfterUse from './FlowModuleDeleteAfterUse.svelte'
+	import { enterpriseLicense } from '$lib/stores'
 
 	export let noEditor: boolean
 	export let flowModule: FlowModule
@@ -70,6 +72,7 @@
 							<Tab value="suspend">Suspend/Approval</Tab>
 							<Tab value="sleep">Sleep</Tab>
 							<Tab value="mock">Mock</Tab>
+							<Tab value="lifetime">Lifetime</Tab>
 							<svelte:fragment slot="content">
 								<div class="overflow-hidden bg-surface">
 									<TabContent value="early-stop" class="flex flex-col flex-1 h-full">
@@ -92,6 +95,11 @@
 											<FlowModuleMock bind:flowModule />
 										</div>
 									</TabContent>
+									<TabContent value="lifetime" class="flex flex-col flex-1 h-full">
+										<div class="p-4 overflow-y-auto">
+											<FlowModuleDeleteAfterUse bind:flowModule disabled={!$enterpriseLicense} />
+										</div>
+									</TabContent>
 								</div>
 							</svelte:fragment>
 						</Tabs>

frontend/src/lib/components/flows/content/FlowBranchesOneWrapper.svelte

@@ -8,10 +8,12 @@
 	import FlowCard from '../common/FlowCard.svelte'
 	import BranchPredicateEditor from './BranchPredicateEditor.svelte'
 	import FlowModuleEarlyStop from './FlowModuleEarlyStop.svelte'
+	import FlowModuleDeleteAfterUse from './FlowModuleDeleteAfterUse.svelte'
 	import FlowModuleSleep from './FlowModuleSleep.svelte'
 	import FlowModuleSuspend from './FlowModuleSuspend.svelte'
 	import SplitPanesWrapper from '../../splitPanes/SplitPanesWrapper.svelte'
 	import FlowModuleMock from './FlowModuleMock.svelte'
+	import { enterpriseLicense } from '$lib/stores'
 	// import FlowRetries from './FlowRetries.svelte'
 
 	export let flowModule: FlowModule
@@ -70,6 +72,7 @@
 							<Tab value="suspend">Suspend/Approval</Tab>
 							<Tab value="sleep">Sleep</Tab>
 							<Tab value="mock">Mock</Tab>
+							<Tab value="lifetime">Lifetime</Tab>
 							<svelte:fragment slot="content">
 								<div class="overflow-hidden bg-surface">
 									<TabContent value="early-stop" class="flex flex-col flex-1 h-full">
@@ -92,6 +95,11 @@
 											<FlowModuleMock bind:flowModule />
 										</div>
 									</TabContent>
+									<TabContent value="lifetime" class="flex flex-col flex-1 h-full">
+										<div class="p-4 overflow-y-auto">
+											<FlowModuleDeleteAfterUse bind:flowModule disabled={!$enterpriseLicense} />
+										</div>
+									</TabContent>
 								</div>
 							</svelte:fragment>
 						</Tabs>

frontend/src/lib/components/flows/content/FlowLoop.svelte

@@ -13,12 +13,14 @@
 	import type { FlowModule } from '$lib/gen/models/FlowModule'
 	import { Pane, Splitpanes } from 'svelte-splitpanes'
 	import { getStepPropPicker } from '../previousResults'
+	import { enterpriseLicense } from '$lib/stores'
 
 	import FlowModuleSleep from './FlowModuleSleep.svelte'
 	import FlowModuleMock from './FlowModuleMock.svelte'
 	import { Play } from 'lucide-svelte'
 	import type { Job } from '$lib/gen'
 	import FlowLoopIterationPreview from '$lib/components/FlowLoopIterationPreview.svelte'
+	import FlowModuleDeleteAfterUse from './FlowModuleDeleteAfterUse.svelte'
 
 	const { previewArgs, flowStateStore, flowStore } =
 		getContext<FlowEditorContext>('FlowEditorContext')
@@ -151,6 +153,7 @@
 					<Tab value="suspend">Suspend/Approval</Tab>
 					<Tab value="sleep">Sleep</Tab>
 					<Tab value="mock">Mock</Tab>
+					<Tab value="lifetime">Lifetime</Tab>
 
 					<svelte:fragment slot="content">
 						<div class="overflow-hidden bg-surface" style="height:calc(100% - 32px);">
@@ -181,6 +184,11 @@
 									<FlowModuleMock bind:flowModule={mod} />
 								</div>
 							</TabContent>
+							<TabContent value="lifetime" class="flex flex-col flex-1 h-full">
+								<div class="p-4 overflow-y-auto">
+									<FlowModuleDeleteAfterUse bind:flowModule={mod} disabled={!$enterpriseLicense} />
+								</div>
+							</TabContent>
 						</div>
 					</svelte:fragment>
 				</Tabs>

frontend/src/lib/components/flows/content/FlowModuleComponent.svelte

@@ -19,6 +19,7 @@
 	import FlowModuleEarlyStop from './FlowModuleEarlyStop.svelte'
 	import FlowModuleSuspend from './FlowModuleSuspend.svelte'
 	import FlowModuleCache from './FlowModuleCache.svelte'
+	import FlowModuleDeleteAfterUse from './FlowModuleDeleteAfterUse.svelte'
 	import FlowRetries from './FlowRetries.svelte'
 	import { getStepPropPicker } from '../previousResults'
 	import { deepEqual } from 'fast-equals'
@@ -70,6 +71,7 @@
 	}
 	let selected = 'inputs'
 	let advancedSelected = 'retries'
+	let advancedRuntimeSelected = 'concurrency'
 	let s3Kind = 'push'
 	let wrapper: HTMLDivElement
 	let panes: HTMLElement
@@ -347,46 +349,30 @@
 								<Tabs bind:selected={advancedSelected}>
 									<Tab value="retries">Retries</Tab>
 									{#if !$selectedId.includes('failure')}
+										<Tab value="runtime">Runtime</Tab>
 										<Tab value="cache">Cache</Tab>
-										<Tab value="concurrency">Concurrency</Tab>
 										<Tab value="early-stop">Early Stop</Tab>
 										<Tab value="suspend">Suspend</Tab>
 										<Tab value="sleep">Sleep</Tab>
 										<Tab value="mock">Mock</Tab>
 										<Tab value="same_worker">Shared Directory</Tab>
-										<Tab value="timeout">Timeout</Tab>
-										<Tab value="priority">Priority</Tab>
 										{#if flowModule.value['language'] === 'python3' || flowModule.value['language'] === 'deno'}
 											<Tab value="s3">S3</Tab>
 										{/if}
 									{/if}
 								</Tabs>
+								{#if advancedSelected === 'runtime'}
+									<Tabs bind:selected={advancedRuntimeSelected}>
+										<Tab value="concurrency">Concurrency</Tab>
+										<Tab value="timeout">Timeout</Tab>
+										<Tab value="priority">Priority</Tab>
+										<Tab value="lifetime">Lifetime</Tab>
+									</Tabs>
+								{/if}
 								<div class="h-[calc(100%-32px)] overflow-auto p-4">
 									{#if advancedSelected === 'retries'}
 										<FlowRetries bind:flowModule />
-									{:else if advancedSelected === 'early-stop'}
-										<FlowModuleEarlyStop bind:flowModule />
-									{:else if advancedSelected === 'suspend'}
-										<div>
-											<FlowModuleSuspend previousModuleId={previousModule?.id} bind:flowModule />
-										</div>
-									{:else if advancedSelected === 'sleep'}
-										<div>
-											<FlowModuleSleep previousModuleId={previousModule?.id} bind:flowModule />
-										</div>
-									{:else if advancedSelected === 'cache'}
-										<div>
-											<FlowModuleCache bind:flowModule />
-										</div>
-									{:else if advancedSelected === 'mock'}
-										<div>
-											<FlowModuleMock bind:flowModule />
-										</div>
-									{:else if advancedSelected === 'timeout'}
-										<div>
-											<FlowModuleTimeout bind:flowModule />
-										</div>
-									{:else if advancedSelected === 'concurrency'}
+									{:else if advancedSelected === 'runtime' && advancedRuntimeSelected === 'concurrency'}
 										<Section label="Concurrency Limits" class="flex flex-col gap-4" eeOnly>
 											<svelte:fragment slot="header">
 												<Tooltip>Allowed concurrency within a given timeframe</Tooltip>
@@ -426,22 +412,11 @@
 												</Alert>
 											{/if}
 										</Section>
-									{:else if advancedSelected === 'same_worker'}
+									{:else if advancedSelected === 'runtime' && advancedRuntimeSelected === 'timeout'}
 										<div>
-											<Alert type="info" title="Share a directory between steps">
-												If shared directory is set, will share a folder that will be mounted on
-												`./shared` for each of them to pass data between each other.
-											</Alert>
-											<Button
-												btnClasses="mt-4"
-												on:click={() => {
-													$selectedId = 'settings-same-worker'
-												}}
-											>
-												Set shared directory in the flow settings
-											</Button>
+											<FlowModuleTimeout bind:flowModule />
 										</div>
-									{:else if advancedSelected === 'priority'}
+									{:else if advancedSelected === 'runtime' && advancedRuntimeSelected === 'priority'}
 										<Section label="Priority" class="flex flex-col gap-4">
 											<!-- TODO: Add EE-only badge when we have it -->
 											<Toggle
@@ -481,6 +456,43 @@
 												</svelte:fragment>
 											</Toggle>
 										</Section>
+									{:else if advancedSelected === 'runtime' && advancedRuntimeSelected === 'lifetime'}
+										<div>
+											<FlowModuleDeleteAfterUse bind:flowModule />
+										</div>
+									{:else if advancedSelected === 'cache'}
+										<div>
+											<FlowModuleCache bind:flowModule />
+										</div>
+									{:else if advancedSelected === 'early-stop'}
+										<FlowModuleEarlyStop bind:flowModule />
+									{:else if advancedSelected === 'suspend'}
+										<div>
+											<FlowModuleSuspend previousModuleId={previousModule?.id} bind:flowModule />
+										</div>
+									{:else if advancedSelected === 'sleep'}
+										<div>
+											<FlowModuleSleep previousModuleId={previousModule?.id} bind:flowModule />
+										</div>
+									{:else if advancedSelected === 'mock'}
+										<div>
+											<FlowModuleMock bind:flowModule />
+										</div>
+									{:else if advancedSelected === 'same_worker'}
+										<div>
+											<Alert type="info" title="Share a directory between steps">
+												If shared directory is set, will share a folder that will be mounted on
+												`./shared` for each of them to pass data between each other.
+											</Alert>
+											<Button
+												btnClasses="mt-4"
+												on:click={() => {
+													$selectedId = 'settings-same-worker'
+												}}
+											>
+												Set shared directory in the flow settings
+											</Button>
+										</div>
 									{:else if advancedSelected === 's3'}
 										<div>
 											<h2 class="pb-4">

frontend/src/lib/components/flows/content/FlowModuleDeleteAfterUse.svelte

@@ -0,0 +1,45 @@
+<script lang="ts">
+	import Toggle from '$lib/components/Toggle.svelte'
+	import Tooltip from '$lib/components/Tooltip.svelte'
+	import type { FlowModule } from '$lib/gen'
+
+	import Section from '$lib/components/Section.svelte'
+
+	export let flowModule: FlowModule
+	export let disabled: boolean = false
+</script>
+
+<Section label="Delete after use">
+	<svelte:fragment slot="header">
+		<Tooltip>
+			The logs, arguments and results of this flow step will be completely deleted from Windmill
+			once the flow is complete. They might be temporarily visible in UI while the flow is running.
+			<br />
+			This also applies to a flow step that has failed: the error will not be accessible.
+			<br />
+			<br />
+			The deletion is irreversible.
+			{#if disabled}
+				<br />
+				<br />
+				This option is only available on Windmill Enterprise Edition.
+			{/if}
+		</Tooltip>
+	</svelte:fragment>
+
+	<Toggle
+		{disabled}
+		size="sm"
+		checked={Boolean(flowModule.delete_after_use)}
+		on:change={() => {
+			if (flowModule.delete_after_use) {
+				flowModule.delete_after_use = undefined
+			} else {
+				flowModule.delete_after_use = true
+			}
+		}}
+		options={{
+			right: 'Delete logs, arguments and results after the flow is complete'
+		}}
+	/>
+</Section>