From 504d01a5631e33b3a2604ef401ab8b73e4a55395 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 24 Sep 2026 11:54:25 +0200 Subject: [PATCH 01/91] feat: rework the git sync setup into a guided flow (#11308) * feat: rework git sync settings into a flat repo list and setup modal Co-Authored-By: Claude Opus 5 (1M context) * feat: add the connect step for gitlab and github in git sync setup Co-Authored-By: Claude Opus 5 (1M context) * feat: open the github app install page from the git sync setup next step Co-Authored-By: Claude Opus 5 (1M context) * feat: let git sync setup set the repository folder Co-Authored-By: Claude Opus 5 (1M context) * feat: add the configure sync step to git sync setup Co-Authored-By: Claude Opus 5 (1M context) * style: even out the git sync filter settings columns Co-Authored-By: Claude Opus 5 (1M context) * style: pin the filter input to the bottom of the filter list Co-Authored-By: Claude Opus 5 (1M context) * style: frame the git sync path filters in a box with an empty hint Co-Authored-By: Claude Opus 5 (1M context) * refactor: use a multiselect for the git sync path filters Co-Authored-By: Claude Opus 5 (1M context) * feat: grey out the skipped connect step in git sync setup Co-Authored-By: Claude Opus 5 (1M context) * style: move the check job line to the bottom of the setup modal Co-Authored-By: Claude Opus 5 (1M context) * fix: hide the check job line once the repository check resolved Co-Authored-By: Claude Opus 5 (1M context) * style: center the repository check loader in the setup modal Co-Authored-By: Claude Opus 5 (1M context) * fix: stop the setup modal shifting when the check job id arrives Co-Authored-By: Claude Opus 5 (1M context) * style: move the existing resource hint to the setup modal footer Co-Authored-By: Claude Opus 5 (1M context) * feat: always show the existing resource option when one exists Co-Authored-By: Claude Opus 5 (1M context) * style: drop the card and collapse from the git sync filter settings Co-Authored-By: Claude Opus 5 (1M context) * style: use a success alert for the pull from git notice Co-Authored-By: Claude Opus 5 (1M context) * feat: make initializing the repository a step of the git sync setup Co-Authored-By: Claude Opus 5 (1M context) * style: hide the push job lines once they succeed Co-Authored-By: Claude Opus 5 (1M context) * style: reword the push step intro Co-Authored-By: Claude Opus 5 (1M context) * style: copy button on the cli snippet and a taller changes list Co-Authored-By: Claude Opus 5 (1M context) * style: match the push step loading to the check step Co-Authored-By: Claude Opus 5 (1M context) * fix: stop the push step shifting when a job id arrives Co-Authored-By: Claude Opus 5 (1M context) * style: let the changes list fill the push step Co-Authored-By: Claude Opus 5 (1M context) * feat: end the git sync setup on a done step instead of a modal Co-Authored-By: Claude Opus 5 (1M context) * feat: delete a git sync repository from a row menu with a confirmation Co-Authored-By: Claude Opus 5 (1M context) * style: lift the git sync done message off the notices Co-Authored-By: Claude Opus 5 (1M context) * feat: restore github app installation transfer between instances Co-Authored-By: Claude Opus 5 (1M context) * fix: address regressions from the git sync setup rework Co-Authored-By: Claude Opus 5 (1M context) * fix: keep the overlay z-index floor in step with the ai chat Co-Authored-By: Claude Opus 5 (1M context) * fix: hold the push state in the setup dialog, not in the step it destroys Co-Authored-By: Claude Opus 5 (1M context) * fix: block every dismissal while setup work is in flight and keep sync reachable Co-Authored-By: Claude Opus 5 (1M context) * fix: regenerate the resource path when the repository changes Co-Authored-By: Claude Opus 5 (1M context) * fix: configure the resource the connect step actually created Co-Authored-By: Claude Opus 5 (1M context) * fix: keep a modal open when a drawer above it takes the click Co-Authored-By: Claude Opus 5 (1M context) * fix: snapshot the topmost surface for keyboard clicks too Co-Authored-By: Claude Opus 5 (1M context) * style: size the add repository buttons and explain git promotion Co-Authored-By: Claude Opus 5 (1M context) * revert: drop the promotion-first action from the empty state Co-Authored-By: Claude Opus 5 (1M context) * style: shrink the test connection result line Co-Authored-By: Claude Opus 5 (1M context) * feat: offer the fork sync and fork pull request toggles during setup Co-Authored-By: Claude Opus 5 (1M context) * style: group promotion repositories under their own label Co-Authored-By: Claude Opus 5 (1M context) * feat: offer the deploy branch pull request toggle during setup Co-Authored-By: Claude Opus 5 (1M context) * fix: wait for the repository credential before saving the setup Co-Authored-By: Claude Opus 5 (1M context) * fix: ignore a superseded credential load and drop the bindable default Co-Authored-By: Claude Opus 5 (1M context) * fix: key setup readiness to the repository it was loaded for Co-Authored-By: Claude Opus 5 (1M context) * fix: tie setup readiness to the attempt that asked for it Co-Authored-By: Claude Opus 5 (1M context) * feat: link to where the access token is created Co-Authored-By: Claude Opus 5 (1M context) * style: say plainly how each provider connects Co-Authored-By: Claude Opus 5 (1M context) * feat: sync resources, variables, schedules and triggers by default Co-Authored-By: Claude Opus 5 (1M context) * style: use the accent colour for the token help link Co-Authored-By: Claude Opus 5 (1M context) * style: move the CI/CD deploy hint next to the pull toggle Co-Authored-By: Claude Opus 5 (1M context) * fix: keep setup gated when the connection could not be read Co-Authored-By: Claude Opus 5 (1M context) * fix: only explain pull-from-git where the workspace can turn it on Co-Authored-By: Claude Opus 5 (1M context) * fix: offer a save retry once the repository is initialized Co-Authored-By: Claude Opus 5 (1M context) * fix: say when pulling fell back to polling on the last setup step Co-Authored-By: Claude Opus 5 (1M context) * fix: clear the initialized flag when a new draft starts Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../src/lib/components/GitDiffPreview.svelte | 7 +- .../common/drawer/Disposable.svelte | 16 +- .../lib/components/common/modal/Modal2.svelte | 172 +++-- .../components/common/stepper/Stepper.svelte | 25 +- .../components/git_sync/GitPushPreview.svelte | 263 ++++++++ .../git_sync/GitSyncContext.svelte.ts | 92 ++- .../git_sync/GitSyncModalManager.svelte | 2 + .../git_sync/GitSyncRepositoryCard.svelte | 96 +-- .../components/git_sync/GitSyncSection.svelte | 426 ++++++++----- .../git_sync/GitSyncSetupModal.svelte | 600 ++++++++++++++++++ .../git_sync/GitSyncSuccessContent.svelte | 72 +++ .../git_sync/GitSyncSuccessModal.svelte | 79 +-- .../git_sync/PushWorkspaceModal.svelte | 374 ++--------- .../git_sync/setup/ConfigureSyncStep.svelte | 314 +++++++++ .../setup/GithubAppConnectForm.svelte | 232 +++++++ .../setup/GitlabProjectConnectForm.svelte | 114 ++++ .../git_sync/setup/TokenUrlConnectForm.svelte | 95 +++ .../git_sync/setup/connectionDefaults.ts | 45 ++ .../git_sync/setup/githubAppSetup.svelte.ts | 50 ++ .../git_sync/setup/repoConnection.test.ts | 29 + .../git_sync/setup/repoConnection.ts | 117 ++++ .../lib/components/select/MultiSelect.svelte | 11 +- .../workspaceSettings/FilterList.svelte | 61 +- .../GitSyncFilterSettings.svelte | 46 +- 24 files changed, 2541 insertions(+), 797 deletions(-) create mode 100644 frontend/src/lib/components/git_sync/GitPushPreview.svelte create mode 100644 frontend/src/lib/components/git_sync/GitSyncSetupModal.svelte create mode 100644 frontend/src/lib/components/git_sync/GitSyncSuccessContent.svelte create mode 100644 frontend/src/lib/components/git_sync/setup/ConfigureSyncStep.svelte create mode 100644 frontend/src/lib/components/git_sync/setup/GithubAppConnectForm.svelte create mode 100644 frontend/src/lib/components/git_sync/setup/GitlabProjectConnectForm.svelte create mode 100644 frontend/src/lib/components/git_sync/setup/TokenUrlConnectForm.svelte create mode 100644 frontend/src/lib/components/git_sync/setup/connectionDefaults.ts create mode 100644 frontend/src/lib/components/git_sync/setup/githubAppSetup.svelte.ts create mode 100644 frontend/src/lib/components/git_sync/setup/repoConnection.test.ts create mode 100644 frontend/src/lib/components/git_sync/setup/repoConnection.ts diff --git a/frontend/src/lib/components/GitDiffPreview.svelte b/frontend/src/lib/components/GitDiffPreview.svelte index 948ae4d4e8..1d18806109 100644 --- a/frontend/src/lib/components/GitDiffPreview.svelte +++ b/frontend/src/lib/components/GitDiffPreview.svelte @@ -1,8 +1,11 @@ -
+
Preview of changes:
{#if !added.length && !deleted.length && !edited.length} diff --git a/frontend/src/lib/components/common/drawer/Disposable.svelte b/frontend/src/lib/components/common/drawer/Disposable.svelte index 642d1366f6..886c14042b 100644 --- a/frontend/src/lib/components/common/drawer/Disposable.svelte +++ b/frontend/src/lib/components/common/drawer/Disposable.svelte @@ -71,6 +71,16 @@ } } + // The entry follows the prop for as long as the overlay is open: a dialog opened before + // the AI chat still has to rise above it when the chat is opened underneath it. + $effect(() => { + const wanted = open && minZIndex > 0 ? minZIndex : undefined + untrack(() => { + if (wanted !== undefined) minZIndexEntries[id] = wanted + else delete minZIndexEntries[id] + }) + }) + // A disposable can be unmounted while still open, by an ancestor that tears its whole // subtree down. Its id would then sit on the stack forever, and since the topmost entry // arbitrates Escape, every overlay opened afterwards would stop answering it. @@ -84,10 +94,10 @@ offset = initialOffset if (stack.val.includes(id)) { stack.val = stack.val.filter((drawer) => drawer !== id) - if (minZIndex > 0) { - delete minZIndexEntries[id] - } } + // Unconditionally: `minZIndex` may have fallen to 0 since the entry was written (the + // chat closed), and a leftover entry keeps every later overlay elevated for good. + delete minZIndexEntries[id] } export function isOpen() { diff --git a/frontend/src/lib/components/common/modal/Modal2.svelte b/frontend/src/lib/components/common/modal/Modal2.svelte index b9e57740b9..6d7c8fdd0f 100644 --- a/frontend/src/lib/components/common/modal/Modal2.svelte +++ b/frontend/src/lib/components/common/modal/Modal2.svelte @@ -10,6 +10,9 @@ import { fade } from 'svelte/transition' import { zIndexes } from '$lib/zIndexes' import { chatState } from '$lib/components/copilot/chat/sharedChatState.svelte' + import Disposable from '../drawer/Disposable.svelte' + import { overlayHostActive, setTopmostSurface } from '../overlayHost.svelte' + import { untrack } from 'svelte' interface Props { title: string @@ -30,6 +33,9 @@ * window, so a stacked pair would both close on one press; set it * false on the underlying modal while its child is up. */ closeOnEscape?: boolean + /** Blocks every way out — the header ✕, Escape and an outside click — for a dialog + * holding work in flight that a dismissal would orphan. */ + preventDismiss?: boolean /** Wider side padding and a lighter title, for a dialog whose body is a form rather * than a list. Opt-in: every other Modal2 keeps the padding and heading it had. */ formStyling?: boolean @@ -51,10 +57,11 @@ contentClasses = '', closeOnOutsideClick = true, closeOnEscape = true, + preventDismiss = false, formStyling = false, headerLeft, headerRight, - children + children: body }: Props = $props() const widthMap = { @@ -84,8 +91,30 @@ isOpen = true } + // Registered in the disposable stack so a drawer opened from inside the dialog (a + // resource picker's editor) stacks above it, and so Escape and outside clicks aimed + // at that drawer do not close the dialog underneath. + let disposable: Disposable | undefined = $state(undefined) + const hostActive = overlayHostActive() + setTopmostSurface(() => disposable?.isTopmost() ?? true) + $effect(() => { + isOpen + untrack(() => (isOpen ? disposable?.openDrawer() : disposable?.closeDrawer())) + }) + + /** Whether this dialog was the overlay on top when the interaction started. `clickOutside` + * awaits its async `exclude` before reporting, and an overlay above closes itself in that + * gap — by the time the report lands, the dialog looks topmost and would close with it. */ + let topmostAtPress = true + function snapshotTopmost() { + topmostAtPress = disposable?.isTopmost() ?? true + } + function handleKeyDown(event: KeyboardEvent) { - if (!isOpen || !closeOnEscape) return + if (!isOpen || !closeOnEscape || preventDismiss) return + // Hidden hosts stay mounted and still receive window keys — see overlayHost. + if (!hostActive()) return + if (!(disposable?.isTopmost() ?? true)) return if (event.key === 'Escape') { event.preventDefault() event.stopPropagation() @@ -97,79 +126,94 @@ return fade(node, { duration: 200 }) } - // Elevate above the AI chat panel (zIndexes.aiChat) while chat is open so - // the dialog isn't hidden behind it; otherwise keep the default modal - // stacking just above disposables (zIndexes.disposables). - const overlayZIndex = $derived( - chatState.size > 0 ? zIndexes.aiChat + 1 : zIndexes.disposables + 10 - ) + // Elevate above the AI chat panel while chat is open so the dialog isn't hidden behind it. + const minZIndex = $derived(chatState.size > 0 ? zIndexes.aiChat + 1 : 0) - + { + // A keyboard-activated control emits a click with no pointer event before it (`detail` + // 0), so the pointerdown snapshot would be whatever a previous interaction left. + if (e.detail === 0) snapshotTopmost() + }} +/> -{#if isOpen} - -
-
+ + {#snippet children({ zIndex })} + {#if isOpen} +
closeOnOutsideClick && close() - }} + class={'fixed top-0 bottom-0 left-0 right-0 transition-all overflow-auto bg-black bg-opacity-60 w-full h-full'} + style="z-index: {zIndex}" + transition:fadeFast|local > - -
- -

{title}

-
+
+
+ closeOnOutsideClick && !preventDismiss && topmostAtPress && close(), + // A dropdown opened from inside the dialog portals its menu out of it, so a + // click on one of its items lands outside this node and would close the + // dialog under the menu. + exclude: async () => + Array.from(document.querySelectorAll('[data-menu]')) as HTMLElement[] + }} + > + +
-
- {@render headerLeft?.()} -
-
- - {@render headerRight?.()} -
- +

{title}

+
+ +
+ {@render headerLeft?.()} +
+
+ + {@render headerRight?.()} +
+ +
+
+ + + +
{})} + > + {@render body?.()} +
- - - -
{})} - > - {@render children?.()} -
- +
-
-
- -{/if} + + {/if} + {/snippet} + diff --git a/frontend/src/lib/components/common/stepper/Stepper.svelte b/frontend/src/lib/components/common/stepper/Stepper.svelte index de6303cedc..5b13d9ddfc 100644 --- a/frontend/src/lib/components/common/stepper/Stepper.svelte +++ b/frontend/src/lib/components/common/stepper/Stepper.svelte @@ -12,6 +12,9 @@ allowStepNavigation?: boolean /** Compact variant, for steering a dialog rather than a full page. */ small?: boolean + /** Steps this run does not go through, greyed out and not clickable — a step the + * chosen path skips rather than one not reached yet. */ + disabledIndices?: number[] } let { @@ -21,7 +24,8 @@ statusByStep = [], hasValidations = false, allowStepNavigation = false, - small = false + small = false, + disabledIndices = [] }: Props = $props() const dispatch = createEventDispatcher() @@ -75,15 +79,20 @@ {#each tabs ?? [] as step, index} + {@const disabled = disabledIndices.includes(index)}
  • { - dispatch('click', { index }) + if (!disabled) dispatch('click', { index }) }} > {#if statusByStep[index] === 'pending'} @@ -94,7 +103,9 @@ small ? 'h-4 w-4 rounded-full flex items-center justify-center text-2xs' : 'h-6 w-6 rounded-full flex items-center justify-center text-xs', - getStepColor(index, selectedIndex, statusByStep, maxReachedIndex) + disabled + ? 'bg-gray-200' + : getStepColor(index, selectedIndex, statusByStep, maxReachedIndex) )} class:font-bold={selectedIndex === index} > @@ -105,7 +116,11 @@ {step} diff --git a/frontend/src/lib/components/git_sync/GitPushPreview.svelte b/frontend/src/lib/components/git_sync/GitPushPreview.svelte new file mode 100644 index 0000000000..bed6d741d1 --- /dev/null +++ b/frontend/src/lib/components/git_sync/GitPushPreview.svelte @@ -0,0 +1,263 @@ + + + +
    + {#if isPreviewLoading} +
    + + Listing the changes to push... +
    + {:else} +

    + Here is what will get pushed to the repository, according to the filters you set. +

    + {/if} + + {#if previewError} + +
    + {previewError} + +
    +     + {/if} + + {#if previewResult && !previewError} +
    +

    Changes to push

    + + {#if previewResult.changes?.length > 0} + + {:else} +
    + The repository already matches this workspace: there is nothing to push. +
    + {/if} +
    + {/if} + + {#if applyError} + + {applyError} + + {/if} + +
    + {#each [{ id: previewJobId, status: previewJobStatus, label: 'Changes job', running: isPreviewLoading }, { id: applyJobId, status: applyJobStatus, label: 'Push job', running: isApplying }] as job (job.label)} + + {#if job.running || (job.id && job.status !== 'success')} +
    + {#if job.status === 'running'} + + {:else if job.status === 'failure'} + + {/if} + {#if job.id} + {job.label}: + + {job.id} + + {/if} +
    + {/if} + {/each} + + +
    +
    + +
    + + {#if showCliInstructions} +
    + + 
    {cliInstructions}
    +
    + {/if} +
    +
    +
    diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index 075c6b1f24..97b79b7c1b 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -1,5 +1,5 @@ import { getContext, setContext } from 'svelte' -import { enterpriseLicense, userStore } from '$lib/stores' +import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores' import { get } from 'svelte/store' import { sendUserToast } from '$lib/toast' import { apiErrorMessage } from '$lib/utils' @@ -46,7 +46,15 @@ export type GitSyncSettings = { export type ModalState = { push: { idx: number; repo: GitSyncRepository; open: boolean } | null pull: { idx: number; repo: GitSyncRepository; open: boolean; settingsOnly?: boolean } | null - success: { open: boolean; savedWithoutInit?: boolean; autoPullOn?: boolean } | null + success: { + open: boolean + savedWithoutInit?: boolean + autoPullOn?: boolean + /** Why pulling fell back to polling, when it did. */ + webhookError?: string + /** Whether this workspace is the one that would turn pulling on for this repository. */ + ownsAutoPull?: boolean + } | null } export type ValidationState = { @@ -202,6 +210,11 @@ export function createGitSyncContext(workspace: string) { 'flow', 'app', 'folder', + 'resource', + 'resourcetype', + 'variable', + 'schedule', + 'trigger', 'workspacedependencies', 'datatablemigration' ] @@ -290,8 +303,23 @@ export function createGitSyncContext(workspace: string) { closeModal('pull') } - function showSuccessModal(savedWithoutInit?: boolean, autoPullOn?: boolean) { - activeModals.success = { open: true, savedWithoutInit, autoPullOn } + /** Only a sync repository in a non-fork workspace has a pull toggle of its own: promotion + * repositories have none, and a fork's pulling is its parent's to set. */ + function ownsAutoPull(repo: GitSyncRepository): boolean { + const ws = get(workspaceStore) + const isFork = + (ws?.startsWith('wm-fork-') ?? false) || + !!get(userWorkspaces)?.find((w) => w.id === ws)?.parent_workspace_id + return !repo.use_individual_branch && !isFork + } + + function showSuccessModal( + savedWithoutInit?: boolean, + autoPullOn?: boolean, + ownsAutoPull?: boolean, + webhookError?: string + ) { + activeModals.success = { open: true, savedWithoutInit, autoPullOn, ownsAutoPull, webhookError } } function closeSuccessModal() { @@ -314,8 +342,11 @@ export function createGitSyncContext(workspace: string) { repo.detectionJobId = undefined repo.detectionJobStatus = undefined - // Track the detection timestamp to avoid race conditions from old jobs + // Track the detection timestamp to avoid race conditions from old jobs. Stamped before + // the request, or a failure to even start the job reads as superseded and leaves the + // repository loading for good. const detectionTimestamp = Date.now() + repo._detectionTimestamp = detectionTimestamp try { const jobId = await JobService.runScriptByPath({ @@ -335,7 +366,6 @@ export function createGitSyncContext(workspace: string) { repo.detectionJobId = jobId repo.detectionJobStatus = 'running' - repo._detectionTimestamp = detectionTimestamp // Use JobManager for polling - result will be the actual job response await jobManager.runWithProgress(() => Promise.resolve(jobId), { @@ -506,7 +536,9 @@ export function createGitSyncContext(workspace: string) { } } - async function saveRepository(idx: number, savedWithoutInit = false) { + /** `announce` off for a caller that reports the save itself — the setup dialog ends on a + * step saying the same thing, and a modal over it would say it twice. */ + async function saveRepository(idx: number, savedWithoutInit = false, announce = true) { const repo = repositories[idx] if (!repo || !validateRepository(repo, idx)) { throw new Error('Cannot save invalid repository') @@ -539,7 +571,8 @@ export function createGitSyncContext(workspace: string) { if (repoToSave.auto_pull) { repoToSave.auto_pull = { ...repoToSave.auto_pull, - enabled_by: repoToSave.auto_pull.enabled ? get(userStore)?.email : undefined + enabled_by: repoToSave.auto_pull.enabled ? get(userStore)?.email : undefined, + webhook_error: await readWebhookError(repoToSave) } } @@ -553,7 +586,31 @@ export function createGitSyncContext(workspace: string) { repoToSave.detectionState = undefined repoToSave.extractedSettings = undefined // Show success modal for new connections - showSuccessModal(savedWithoutInit, repoToSave.auto_pull?.enabled === true) + if (announce) { + showSuccessModal( + savedWithoutInit, + repoToSave.auto_pull?.enabled === true, + ownsAutoPull(repoToSave), + repoToSave.auto_pull?.webhook_error + ) + } + } + } + + /** The webhook is registered after the settings commit, best-effort, and a failure is + * reported only through the stored settings — so it stays invisible until they are read + * back. Without this, pulling reads as working when it has silently fallen back to + * polling (an app missing the webhook permission, an instance GitHub cannot reach). */ + async function readWebhookError(repo: GitSyncRepository): Promise { + if (!repo.auto_pull?.enabled) return undefined + try { + const settings = await WorkspaceService.getSettings({ workspace }) + return settings.git_sync?.repositories?.find( + (r) => r.git_repo_resource_path.replace('$res:', '') === repo.git_repo_resource_path + )?.auto_pull?.webhook_error + } catch { + // Only costs the warning; the card shows it on the next load. + return undefined } } @@ -704,6 +761,11 @@ export function createGitSyncContext(workspace: string) { 'flow', 'app', 'folder', + 'resource', + 'resourcetype', + 'variable', + 'schedule', + 'trigger', 'workspacedependencies', 'datatablemigration' ] @@ -712,9 +774,10 @@ export function createGitSyncContext(workspace: string) { legacyImported: false, isUnsavedConnection: true, collapsed: false - // Pull-from-Git defaults are applied by the repository card once the - // selected resource resolves: only app-backed repos (instant webhook - // delivery) default to auto-pull on; polling is opt-in for token repos. + // Pull-from-Git defaults are applied once the selected resource resolves, by + // whoever is configuring the connection (see applyNewConnectionDefaults): only + // app-backed repos (instant webhook delivery) default to auto-pull on; polling is + // opt-in for token repos. }) gitSyncTestJobs.push({ jobId: '', @@ -741,6 +804,11 @@ export function createGitSyncContext(workspace: string) { 'flow', 'app', 'folder', + 'resource', + 'resourcetype', + 'variable', + 'schedule', + 'trigger', 'workspacedependencies', 'datatablemigration' ] diff --git a/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte b/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte index 610dafb562..cb8236cafc 100644 --- a/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncModalManager.svelte @@ -111,5 +111,7 @@ bind:open={gitSyncContext.activeModals.success.open} savedWithoutInit={gitSyncContext.activeModals.success.savedWithoutInit} autoPullOn={gitSyncContext.activeModals.success.autoPullOn} + explainAutoPullOff={gitSyncContext.activeModals.success.ownsAutoPull ?? false} + webhookError={gitSyncContext.activeModals.success.webhookError} /> {/if} diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index 291c89c606..8c3af1c54d 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -1,7 +1,6 @@ {#if !gitSyncContext} @@ -132,7 +228,7 @@ {#snippet actions()} {#if (gitSyncAllowed || gitSyncStatus.user_count != null) && gitSyncContext?.repositories != undefined} +
    • + {:else} +
    + {#each syncRows as { repo, idx } (idx)} + {@render repoRow(repo, idx)} + {/each} + {#if promotionRows.length > 0} + +
    Promotion repositories
    + {#each promotionRows as { repo, idx } (idx)} + {@render repoRow(repo, idx)} + {/each} + {/if} +
    - {#if $enterpriseLicense} - - {#if primarySync && !primarySync.repo?.isUnsavedConnection && !devSingleRepo} - {#if secondarySync.length > 0 || secondarySyncExpanded} -
    - - - {#if secondarySyncExpanded} -
    - {#if secondarySync.length === 0} -
    - No secondary sync repositories configured -
    - {:else} - {#each secondarySync as { repo, idx } (repo.git_repo_resource_path)} -
    - -
    - {/each} - {/if} - - {#if !hasUnsavedSecondary} -
    - -
    - {/if} -
    - {/if} -
    - {:else} - - {#if !hasUnsavedSecondary} -
    - + {addSyncLabel} + + {#if !$enterpriseLicense}{/if}
    {/if} - {/if} - {/if} - - - {#if showPromotion && !devSingleRepo} -
    - gitSyncContext.addPromotionRepository()} - isCollapsible={false} - showEmptyState={primaryPromotion?.repo === null} - /> - - - {#if primaryPromotion && !primaryPromotion.repo?.isUnsavedConnection} - {#if secondaryPromotion.length > 0 || secondaryPromotionExpanded} -
    - - - {#if secondaryPromotionExpanded} -
    - {#if secondaryPromotion.length === 0} -
    - No secondary promotion repositories configured -
    - {:else} - {#each secondaryPromotion as { repo, idx } (repo.git_repo_resource_path)} -
    - -
    - {/each} - {/if} - - {#if !hasUnsavedSecondaryPromotion} -
    - -
    - {/if} -
    - {/if} -
    - {:else} - - {#if !hasUnsavedSecondaryPromotion} -
    - -
    - {/if} - {/if} + {#if showAddPromotion} +
    + + + Each deploy in this workspace pushes its changes to a dedicated + wm_deploy/** + branch of the repository instead of committing to its tracked branch directly. Merging + that branch promotes the change: the workspace that syncs the tracked branch deploys + it on merge, so set up Git Sync there. Windmill can open the pull request for each + deploy branch, or you can use the open-pr-on-commit workflow. + + {#if !$enterpriseLicense}{/if} +
    {/if}
    - {:else if !showPromotion} + {/if} + + {#if $enterpriseLicense && !showPromotion}
    + (deleting = undefined)} + > + + Deploys of this workspace will stop being committed to + {deleting?.path}. The repository itself and its resource are + left untouched. + + + + + + (settingsPath = undefined)} + > + settingsDrawer?.closeDrawer()} + > + {#if settingsIdx !== -1} + {#key settingsIdx} + + {/key} + {/if} + + + {/if} {/if} + +{#snippet repoRow(repo: GitSyncRepository, idx: number)} + {@const validation = gitSyncContext?.getValidation(idx)} +
    +
    +
    + + {repo.git_repo_resource_path || 'No resource selected'} + + {#if repo.isUnsavedConnection} + Not saved + {:else if validation?.hasChanges} + Unsaved changes + {/if} + {#if repo.legacyImported} + Legacy configuration + {/if} +
    + {rowLabel(idx)} +
    +
    + + (deleting = { idx, path: repo.git_repo_resource_path }) + } + ]} + /> +
    +
    +{/snippet} diff --git a/frontend/src/lib/components/git_sync/GitSyncSetupModal.svelte b/frontend/src/lib/components/git_sync/GitSyncSetupModal.svelte new file mode 100644 index 0000000000..c89ed45d76 --- /dev/null +++ b/frontend/src/lib/components/git_sync/GitSyncSetupModal.svelte @@ -0,0 +1,600 @@ + + + +
    + { + if (e.detail.index === 0 && step > 1) back() + }} + /> + +
    +
    + {#if step === 1} + {#snippet githubIcon()} + + {/snippet} + {#snippet gitlabIcon()} + + {/snippet} + {#snippet resourceIcon()} + + {/snippet} +
    + {@render providerCard( + 'github_app', + githubIcon, + 'GitHub App (recommended)', + 'Connect with the GitHub UI.', + !$enterpriseLicense + )} + {@render providerCard( + 'github_pat', + githubIcon, + 'GitHub personal access token', + 'Generate a token from GitHub to allow Windmill to access your account.' + )} + {@render providerCard( + 'gitlab', + gitlabIcon, + 'GitLab', + 'Generate a token from GitLab to allow Windmill to access your account.' + )} + {#if hasResource || provider === 'existing'} + {@render providerCard( + 'existing', + resourceIcon, + 'Use an existing git_repository resource', + 'Pick a resource that already holds the repository URL and its credentials.' + )} + {/if} +
    + {#if provider === 'existing'} + + {/if} + {:else if step === 5} + {#if done} + + {/if} + {:else if step === 4} + {#if draft} + (applying = v)} + onSuccess={() => { + initialized = true + void saveDraft() + }} + /> + {/if} + {#if saveError} + + {saveError} + + {/if} + {:else if step === 3} + {#if draftIdx !== -1} + + {#key configureAttempt} + + {/key} + {/if} + {#if saveError} + + {saveError} + + {/if} + {:else if $workspaceStore} +
    + {#if provider === 'github_app' && githubApp} + + {:else if gitlabHeld} + + {:else if provider === 'gitlab'} + + {:else} + + {/if} + + {#if connection} +
    + + +
    + Save as resource + + Windmill keeps the repository and its access as a git_repository resource + at this path. + + {#key connection.url} + + {/key} +
    +
    + {/if} + + {#if connectError} + + {connectError} + + {/if} +
    + {/if} +
    + +
    +
    + {#if step > 1 && step < 5} + + {:else if step === 1 && !hasResource && provider !== 'existing'} + + {/if} +
    + {#if step === 5} + + {:else if step === 4} +
    + {#if initialized} + + {:else} + + + {/if} +
    + {:else if step === 3} + {#if draft?.detectionState === 'no-wmill'} + + {:else} + + {/if} + {:else} + + {/if} +
    +
    +
    +     + +{#snippet providerCard( + key: Provider, + icon: Snippet, + title: string, + subtitle: string, + eeOnly: boolean = false +)} + {#snippet description()} + + {subtitle} + {#if eeOnly}{/if} + + {/snippet} + selectProvider(key)} + /> +{/snippet} diff --git a/frontend/src/lib/components/git_sync/GitSyncSuccessContent.svelte b/frontend/src/lib/components/git_sync/GitSyncSuccessContent.svelte new file mode 100644 index 0000000000..047f55ef41 --- /dev/null +++ b/frontend/src/lib/components/git_sync/GitSyncSuccessContent.svelte @@ -0,0 +1,72 @@ + + + + +
    + {#if savedWithoutInit} + + Only new changes will be pushed to this repository. What this workspace holds today has not + been pushed to it. + + {/if} + + {#if autoPullOn && webhookError} + + {webhookError} + + {:else if autoPullOn} + + New commits to the tracked branch deploy into this workspace automatically. You can adjust + this anytime on the repository's settings. + + {:else if explainAutoPullOff} + + Turn on "Automatically deploy changes from Git" on the repository to have Windmill pull new + commits into this workspace for you. Prefer to control deployment from your own pipeline + (tests, custom gating, deploy on PR merge)? Set up GitHub Actions or a similar CI/CD workflow + instead. + + {/if} + + +
    + +
    +

    Git sync connection saved successfully!

    +

    + Your repository is now configured to receive changes from Windmill. +

    +
    +
    +
    diff --git a/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte b/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte index 32dc68a0f4..00ffa285b2 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSuccessModal.svelte @@ -1,81 +1,24 @@ -
    - -
    -
    - -
    -
    -

    Git sync connection saved successfully!

    -

    Your repository is now configured to receive changes from Windmill.

    -
    -
    - - - {#if savedWithoutInit} -
    -

    Repository saved without initialization

    -

    - Only new changes will be pushed to this repository. Existing content in Windmill has not - been initialized to the repository. -

    -
    - {/if} - - - {#if autoPullOn} -
    -

    - - Pull from Git is on -

    -

    - New commits to the tracked branch deploy into this workspace automatically. You can - adjust this anytime on the repository card. -

    -
    - {:else} -
    -

    - - Deploy changes from Git back to Windmill -

    -

    - Turn on "Automatically deploy changes from Git" on the repository to have Windmill pull - new commits into this workspace for you. -

    -
    -

    - Prefer to control deployment from your own pipeline (tests, custom gating, deploy on PR - merge)? Set up GitHub Actions or similar CI/CD workflows instead. -

    -
    - - - Learn more about CI-based deployment - -
    -
    -
    - {/if} -
    +     diff --git a/frontend/src/lib/components/git_sync/PushWorkspaceModal.svelte b/frontend/src/lib/components/git_sync/PushWorkspaceModal.svelte index 87c6ecb807..dea331bea5 100644 --- a/frontend/src/lib/components/git_sync/PushWorkspaceModal.svelte +++ b/frontend/src/lib/components/git_sync/PushWorkspaceModal.svelte @@ -1,13 +1,9 @@ -
    - -

    Push your current workspace content to the connected Git repository based on the configured - filters.

    -

    Note: This will not update git sync settings in wmill.yaml. Settings can only be pulled from - the repository as it is the source of truth.

    - - - {#if isNewConnection} -
    -

    Settings that will be pushed to repository

    -
    -
    Include paths: {uiState.include_path?.join(', ') || 'None'}
    -
    Exclude paths: {uiState.exclude_path?.join(', ') || 'None'}
    - {#if uiState.extra_include_path?.length > 0} -
    Extra include paths: {uiState.extra_include_path.join(', ')}
    - {/if} -
    Include types: {uiState.include_type?.join(', ') || 'None'}
    -
    -

    To modify these settings, cancel and configure them in the workspace settings.

    -
    + +
    +
    Include paths: {uiState.include_path?.join(', ') || 'None'}
    +
    Exclude paths: {uiState.exclude_path?.join(', ') || 'None'}
    + {#if uiState.extra_include_path?.length} +
    Extra include paths: {uiState.extra_include_path.join(', ')}
    {/if} - - - {#if !previewResult} -
    - -
    - {/if} - - - {#if previewJobId} -
    - {#if previewJobStatus === 'running'} - - {:else if previewJobStatus === 'success'} - - {:else if previewJobStatus === 'failure'} - - {/if} - Preview job: - - {previewJobId} - -
    - {/if} - - - {#if previewError} - - {previewError} - - {/if} - - - {#if previewResult && !previewError} -
    -

    Changes to Push

    - - {#if previewResult.changes?.length > 0} - - {:else} -
    -
    No changes to push to the repository.
    -
    - {/if} -
    - {/if} - - - {#if previewResult && !previewError} - {@const hasChanges = previewResult.changes?.length > 0} - {#if hasChanges} -
    -
    - - {#if isNewConnection && onSaveWithoutInit} - - {/if} -
    -
    - {/if} - {/if} - - - {#if applyJobId} -
    - {#if applyJobStatus === 'running'} - - {:else if applyJobStatus === 'success'} - - {:else if applyJobStatus === 'failure'} - - {/if} - Push job: - - {applyJobId} - -
    - {/if} - - - {#if applyError} - - {applyError} - - {/if} - - -
    - - - {#if showCliInstructions} -
    - 
    -# Setup (only needed if local folder not initialized yet)
-npm install -g windmill-cli
-wmill workspace add {$workspaceStore} {$workspaceStore} {window.location.origin}
-wmill init --workspace {$workspaceStore} --repository {gitRepoResourcePath}
-
-# Pull workspace content to git repository
-wmill sync pull --workspace {$workspaceStore} --repository {gitRepoResourcePath}
    -
    - {/if} -
    +
    Included types: {uiState.include_type?.join(', ') || 'None'}
    +

    + This does not update the git sync settings in wmill.yaml: they + can only be pulled from the repository, which is their source of truth. To change what is + pushed, cancel and edit the filters in the workspace settings. +

    + + {#if open} + {#key gitRepoResourcePath} + + {/key} + {/if} + + + {#snippet actions()} + + {#if isNewConnection && onSaveWithoutInit} + + {/if} + {/snippet} diff --git a/frontend/src/lib/components/git_sync/setup/ConfigureSyncStep.svelte b/frontend/src/lib/components/git_sync/setup/ConfigureSyncStep.svelte new file mode 100644 index 0000000000..c1ff4e1f0d --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/ConfigureSyncStep.svelte @@ -0,0 +1,314 @@ + + +{#if repo} +
    + {#if factsError} + +
    + {factsError} + +
    +     + {/if} + {#if !repo.detectionState || repo.detectionState === 'idle' || repo.detectionState === 'loading'} +
    + + Checking the repository... +
    + {:else if repo.detectionState === 'error'} + +
    + {repo.detectionError || 'The check failed.'} + Check the URL and the token, or go back and connect the repository again. + +
    +     + {:else if repo.detectionState === 'no-wmill'} + + The repository has no Windmill configuration yet. Choose what to sync, then initialize it + with the current content of this workspace. + + {:else if repo.detectionState === 'has-wmill'} + + The repository already holds a Windmill configuration. Its sync settings are loaded below. + + {/if} + + {#if repo.detectionState === 'no-wmill' || repo.detectionState === 'has-wmill'} + + {#snippet subtitle()} + + {/snippet} + + + {#if mode === 'promotion'} + + {#if managedCredential} + setPromotionOpenPrs(e.detail)} + /> + {:else} + + To open a pull request for each deploy branch, connect the repository through the GitHub + App or give a GitLab repository a project access token — otherwise set up the + open-pr-on-commit workflow in the repository. + + {/if} + {:else if !isFork} +
    + setAutoPull(e.detail)} + > + {#snippet right()} + {#if !$enterpriseLicense}{/if} + {/snippet} + + {#if repo.auto_pull?.enabled} + + {managedCredential + ? 'New commits are delivered instantly by webhook, with polling as a fallback.' + : 'The branch is checked for new commits about every minute.'} + + {:else} + + Leave it off to control deployment from your own pipeline (tests, custom gating, + deploy on PR merge) with + GitHub Actions or a similar CI/CD workflow. + + {/if} + setSyncForks(e.detail)} + /> + {#if managedCredential} + setForkOpenPrs(e.detail)} + > + {#snippet right()} + {#if !$enterpriseLicense}{/if} + {/snippet} + + {/if} +
    + {/if} + {/if} + + {#if repo.detectionState !== 'no-wmill' && repo.detectionState !== 'has-wmill'} + +
    + {#if repo.detectionJobId} + {#if repo.detectionJobStatus === 'success'} + + {:else if repo.detectionJobStatus === 'failure'} + + {/if} + Check job: + + {repo.detectionJobId} + + {/if} +
    + {/if} +
    +{/if} diff --git a/frontend/src/lib/components/git_sync/setup/GithubAppConnectForm.svelte b/frontend/src/lib/components/git_sync/setup/GithubAppConnectForm.svelte new file mode 100644 index 0000000000..fdea233345 --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/GithubAppConnectForm.svelte @@ -0,0 +1,232 @@ + + +
    + {#if (!setup.loaded || gh.loadingGithubInstallations) && gh.workspaceGithubInstallations.length === 0} +
    + Loading GitHub App installations... +
    + {:else} + {#if usable.length === 0} + + {#if gh.isCheckingInstallation} + Finish installing the Windmill GitHub App in the tab that opened. This dialog picks up the + installation on its own. + {:else} + Install the Windmill GitHub App on the GitHub account or organization that owns the + repository, then come back here. + {/if} + + {:else} + + {/if} +
    diff --git a/frontend/src/lib/components/git_sync/setup/TokenUrlConnectForm.svelte b/frontend/src/lib/components/git_sync/setup/TokenUrlConnectForm.svelte new file mode 100644 index 0000000000..2d8c7f2f48 --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/TokenUrlConnectForm.svelte @@ -0,0 +1,95 @@ + + +
    + + +
    diff --git a/frontend/src/lib/components/git_sync/setup/connectionDefaults.ts b/frontend/src/lib/components/git_sync/setup/connectionDefaults.ts new file mode 100644 index 0000000000..4af90decf6 --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/connectionDefaults.ts @@ -0,0 +1,45 @@ +import type { GitSyncRepository } from '../GitSyncContext.svelte' + +/** + * Defaults for a connection that has not been saved yet, once its resource is known. + * Only fills what the user has not set, so it is safe to call again. + */ +export function applyNewConnectionDefaults( + repo: GitSyncRepository, + o: { + mode: 'sync' | 'promotion' + /** Windmill holds a credential for the repository: a GitHub App or a stored token. */ + managedCredential: boolean + isGithubApp: boolean + isFork: boolean + ee: boolean + } +): void { + if (!repo.isUnsavedConnection) return + // Pulling from Git defaults on only where Windmill can receive webhooks. Polling is + // opt-in for token repositories, and forks never get the parent-only defaults (the + // backend rejects them). + if ( + o.mode === 'sync' && + o.managedCredential && + !o.isFork && + o.ee && + repo.auto_pull === undefined + ) { + repo.auto_pull = { enabled: true, mode: 'auto', sync_forks: true } + } + // A promotion deploy pushes a wm_deploy/** branch that exists to be merged; without a + // pull request it is an orphaned branch. Fork PRs stay opt-in everywhere. + if ( + o.mode === 'promotion' && + o.managedCredential && + o.ee && + repo.promotion_open_prs === undefined + ) { + repo.promotion_open_prs = true + } + // Webhook with a polling fallback is the only delivery for app repositories. + if (o.isGithubApp && repo.auto_pull?.mode === 'polling') { + repo.auto_pull = { ...repo.auto_pull, mode: 'auto' } + } +} diff --git a/frontend/src/lib/components/git_sync/setup/githubAppSetup.svelte.ts b/frontend/src/lib/components/git_sync/setup/githubAppSetup.svelte.ts new file mode 100644 index 0000000000..645973720f --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/githubAppSetup.svelte.ts @@ -0,0 +1,50 @@ +import { + createGitHubAppState, + loadGithubInstallations, + startInstallationCheck, + stopInstallationCheck, + type GitHubAppState +} from '$lib/githubApp' + +/** + * GitHub App installations for the setup dialog. Owned by the dialog rather than the + * connect form so the installations are known before the user leaves step 1: the + * install page has to be opened synchronously from that click, or the browser blocks + * the new tab. + */ +export class GithubAppSetup { + gh: GitHubAppState = $state(createGitHubAppState()) + loaded = $state(false) + + constructor(private workspace: string) {} + + get usable() { + return this.gh.workspaceGithubInstallations.filter((i) => !i.error) + } + + async reload(): Promise { + try { + await loadGithubInstallations(this.gh, this.workspace) + } catch { + // loadGithubInstallations toasts its own failure + } + this.loaded = true + const usable = this.usable + if (!usable.some((i) => i.installation_id === this.gh.selectedGHAppInstallationId)) { + this.gh.selectedGHAppInstallationId = usable[0]?.installation_id + this.gh.selectedGHAppRepository = undefined + } + } + + /** Must be called from a click handler, synchronously. */ + openInstall(): void { + const url = this.gh.githubInstallationUrl + if (!url) return + window.open(url, '_blank', 'noopener') + startInstallationCheck(this.gh, this.workspace, () => void this.reload()) + } + + dispose(): void { + stopInstallationCheck(this.gh) + } +} diff --git a/frontend/src/lib/components/git_sync/setup/repoConnection.test.ts b/frontend/src/lib/components/git_sync/setup/repoConnection.test.ts new file mode 100644 index 0000000000..4b9cdbe9cb --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/repoConnection.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from 'vitest' +import { parseRepoUrl, repoSlug, withToken } from './repoConnection' + +describe('parseRepoUrl', () => { + test('drops credentials, query and fragment from a pasted URL', () => { + const url = parseRepoUrl(' https://someone:secret@github.com/acme/infra.git?utm=1#readme ') + expect(url?.toString()).toBe('https://github.com/acme/infra.git') + }) + + test('refuses what cannot carry a token safely or name a repository', () => { + // http would put the token on the wire, and the rest name no repository at all. + expect(parseRepoUrl('http://github.com/acme/infra')).toBeUndefined() + expect(parseRepoUrl('ssh://git@github.com/acme/infra')).toBeUndefined() + expect(parseRepoUrl('github.com/acme/infra')).toBeUndefined() + expect(parseRepoUrl('https://github.com/acme')).toBeUndefined() + }) +}) + +test('withToken encodes the token so a punctuated one survives the round trip', () => { + const token = 'glpat-a/b@c:d' + const embedded = withToken(parseRepoUrl('https://gitlab.com/acme/infra')!, token, 'gitlab') + expect(embedded).toBe(`https://oauth2:${encodeURIComponent(token)}@gitlab.com/acme/infra`) + expect(new URL(embedded).password).toBe(encodeURIComponent(token)) +}) + +test('repoSlug names the resource after the repository', () => { + expect(repoSlug('https://github.com/acme/My-Repo.git')).toBe('my_repo') + expect(repoSlug('https://gitlab.com/group/sub/project/')).toBe('project') +}) diff --git a/frontend/src/lib/components/git_sync/setup/repoConnection.ts b/frontend/src/lib/components/git_sync/setup/repoConnection.ts new file mode 100644 index 0000000000..81955086fd --- /dev/null +++ b/frontend/src/lib/components/git_sync/setup/repoConnection.ts @@ -0,0 +1,117 @@ +import { GitSyncService, ResourceService, VariableService } from '$lib/gen' + +/** What a provider form hands back once the user has picked a repository. */ +export type RepoConnection = { + /** The repository URL, never carrying a secret. */ + url: string + isGithubApp?: boolean + /** A token for Windmill to store and renew server-side (EE GitLab). */ + heldToken?: string + /** `url` with the token embedded, filed as a secret variable because the + * resource value itself is readable by anyone who can read the resource. */ + tokenUrl?: string +} + +/** Username git sends alongside a token: GitHub ignores it for PATs, GitLab + * requires a non-empty one and documents `oauth2`. */ +const TOKEN_USERNAME = { github: 'x-access-token', gitlab: 'oauth2' } as const + +export type TokenProvider = keyof typeof TOKEN_USERNAME + +/** Parses a pasted repository URL, dropping any credentials it carried. */ +export function parseRepoUrl(raw: string): URL | undefined { + let u: URL + try { + u = new URL(raw.trim()) + } catch { + return undefined + } + // https only: the token is embedded in this URL, and plain http would put it on the wire. + if (u.protocol !== 'https:') return undefined + if (u.pathname.replace(/\/+$/, '').split('/').filter(Boolean).length < 2) return undefined + u.username = '' + u.password = '' + u.search = '' + u.hash = '' + return u +} + +export function withToken(url: URL, token: string, provider: TokenProvider): string { + const u = new URL(url.toString()) + u.username = TOKEN_USERNAME[provider] + u.password = token.trim() + return u.toString() +} + +/** A resource name derived from the repository, e.g. `acme/infra.git` → `infra`. */ +export function repoSlug(url: string): string { + const last = + url + .replace(/\.git\/?$/, '') + .replace(/\/+$/, '') + .split('/') + .pop() ?? '' + return last.replace(/[^a-zA-Z0-9_]/g, '_').toLowerCase() || 'git_repository' +} + +/** + * Writes everything a connection needs and creates the `git_repository` resource at + * `path`. A token-bearing URL goes to a secret variable at the same path, which the + * backend renames along with the resource. + */ +export async function createRepositoryResource( + workspace: string, + path: string, + conn: RepoConnection, + location: { branch?: string; folder?: string } +): Promise { + const branch = location.branch?.trim() + // Relative to the repository root, which is how the sync script joins it. + const folder = location.folder?.trim().replace(/^\/+|\/+$/g, '') + let url = conn.url + let createdVariable = false + if (conn.tokenUrl) { + if (await VariableService.existsVariable({ workspace, path })) { + throw new Error(`A variable already exists at ${path}. Pick another path.`) + } + await VariableService.createVariable({ + workspace, + requestBody: { + path, + value: conn.tokenUrl, + is_secret: true, + description: `Authenticated URL of the git repository ${conn.url}` + } + }) + createdVariable = true + url = `$var:${path}` + } + try { + if (conn.heldToken) { + await GitSyncService.setGitCredential({ + workspace, + requestBody: { repo_url: conn.url, token: conn.heldToken } + }) + } + await ResourceService.createResource({ + workspace, + requestBody: { + path, + resource_type: 'git_repository', + description: `Git repository ${conn.url}`, + value: { + url, + ...(branch ? { branch } : {}), + ...(folder ? { folder } : {}), + is_github_app: !!conn.isGithubApp + } + } + }) + } catch (e) { + // Otherwise a retry at the same path is refused over this run's own secret. + if (createdVariable) { + await VariableService.deleteVariable({ workspace, path }).catch(() => {}) + } + throw e + } +} diff --git a/frontend/src/lib/components/select/MultiSelect.svelte b/frontend/src/lib/components/select/MultiSelect.svelte index 419984b20d..0ebb824865 100644 --- a/frontend/src/lib/components/select/MultiSelect.svelte +++ b/frontend/src/lib/components/select/MultiSelect.svelte @@ -18,6 +18,7 @@ let { items, placeholder = 'Select items', + searchPlaceholder = 'Search...', value = $bindable(), class: className = '', style, @@ -42,6 +43,9 @@ items?: Item[] value: Value[] placeholder?: string + /** Placeholder of the dropdown's search field. Worth setting where the list is there to + * create entries rather than pick from, so the field says what to type. */ + searchPlaceholder?: string class?: string style?: string filterText?: string @@ -96,7 +100,12 @@ function onAddValue(item: ProcessedItem) { if (item.__is_create && onCreateItem) { + const before = currentValue.length onCreateItem(item.value) + // Cleared only once the entry exists: it is then in the list, and leaving it in the + // field would filter the list down to it. A consumer that refused the value (a + // duplicate, a name it rejects) keeps what was typed, to be corrected. + if ((value?.length ?? 0) !== before) filterText = '' } else { value = [...currentValue, item.value] } @@ -192,7 +201,7 @@ bind:value={filterText} inputProps={{ onblur: (e) => (e.preventDefault(), searchInputEl?.focus()), - placeholder: 'Search...' + placeholder: searchPlaceholder }} class="!pr-7 !bg-surface" /> diff --git a/frontend/src/lib/components/workspaceSettings/FilterList.svelte b/frontend/src/lib/components/workspaceSettings/FilterList.svelte index fccc59681d..599015aa49 100644 --- a/frontend/src/lib/components/workspaceSettings/FilterList.svelte +++ b/frontend/src/lib/components/workspaceSettings/FilterList.svelte @@ -1,5 +1,6 @@ @@ -33,34 +26,14 @@ {/if}
    -
    - {#each items as item, idx (item)} - - {item} - - - {/each} - - (newItem = e.currentTarget.value)} - onkeydown={(e) => e.key === 'Enter' && (addItem(), e.preventDefault())} - /> - -
    +
    diff --git a/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte b/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte index dcbef04a51..87409a8134 100644 --- a/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/GitSyncFilterSettings.svelte @@ -1,13 +1,10 @@ -
    - -
    +
    +
    - - Git Sync filter settings +

    {title}

    {#if isLegacyRepo} This repository uses legacy configuration format and inherits settings from @@ -142,22 +136,14 @@ {/if}
    -
    - {#if !collapsed} {#if isEditable} -
    -
    +
    +
    @@ -337,14 +323,9 @@
    -
    -
    - {isInitialSetup ? 'Configure initial sync settings' : 'Review migration settings'} -
    -
    {:else} -
    +
    @@ -454,5 +435,4 @@ git push
    {/if} - {/if}
    From 80903c9a64125763e7514c09618e1def21a5bf97 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 24 Sep 2026 12:58:59 +0200 Subject: [PATCH 02/91] feat: per-folder AI chat instructions via ai_instruction resources (#11325) * feat: give the AI chat per-folder instructions from ai_instruction resources Co-Authored-By: Claude Opus 5.5 (1M context) * fix: hold a change back until the model has read its folder instructions Co-Authored-By: Claude Opus 5.5 (1M context) * fix: find nested trigger paths and scope instruction deliveries by workspace Co-Authored-By: Claude Opus 5.5 (1M context) * fix: cap folder instructions per result and record holds as their own outcome Co-Authored-By: Claude Opus 5.5 (1M context) * fix: show a held call as its own row and cover group folders Co-Authored-By: Claude Opus 5.5 (1M context) * fix: state how folder instructions combine with workspace and user ones Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- ...3023_ai_instruction_resource_type.down.sql | 6 + ...173023_ai_instruction_resource_type.up.sql | 15 ++ docs/auth-surface.md | 6 + .../copilot/chat/AIChatManager.svelte.ts | 49 +++- .../copilot/chat/ToolExecutionDisplay.svelte | 15 +- .../lib/components/copilot/chat/anthropic.ts | 3 +- .../copilot/chat/folderInstructions.ts | 220 ++++++++++++++++++ .../copilot/chat/global/globalAssembly.ts | 3 + .../copilot/chat/openai-responses.ts | 3 +- .../components/copilot/chat/shared.test.ts | 105 ++++++++- .../src/lib/components/copilot/chat/shared.ts | 58 ++++- frontend/src/lib/components/copilot/lib.ts | 3 +- 12 files changed, 473 insertions(+), 13 deletions(-) create mode 100644 backend/migrations/20260923173023_ai_instruction_resource_type.down.sql create mode 100644 backend/migrations/20260923173023_ai_instruction_resource_type.up.sql create mode 100644 frontend/src/lib/components/copilot/chat/folderInstructions.ts diff --git a/backend/migrations/20260923173023_ai_instruction_resource_type.down.sql b/backend/migrations/20260923173023_ai_instruction_resource_type.down.sql new file mode 100644 index 0000000000..25b81b8c69 --- /dev/null +++ b/backend/migrations/20260923173023_ai_instruction_resource_type.down.sql @@ -0,0 +1,6 @@ +-- `created_by` tells this migration's row from one a user created by hand; a hub +-- sync updates in place and leaves it alone. +DELETE FROM resource_type +WHERE workspace_id = 'admins' + AND name = 'ai_instruction' + AND created_by = 'system'; diff --git a/backend/migrations/20260923173023_ai_instruction_resource_type.up.sql b/backend/migrations/20260923173023_ai_instruction_resource_type.up.sql new file mode 100644 index 0000000000..1b89469a91 --- /dev/null +++ b/backend/migrations/20260923173023_ai_instruction_resource_type.up.sql @@ -0,0 +1,15 @@ +-- Folder instructions for the AI chat: a markdown file resource whose +-- `value.content` applies to every item under the directory it lives in. +-- `format_extension` makes the resource editor render it as a plain .md file. +-- Seeded under 'admins' so every workspace sees it before the next hub sync. +INSERT INTO resource_type (workspace_id, name, schema, description, created_by, format_extension, edited_at) +VALUES ( + 'admins', + 'ai_instruction', + '{"type": "object", "properties": {"content": {"type": "string"}}}', + 'Instructions for the AI chat that apply to every item under the folder this resource lives in, like an AGENTS.md: an ai_instruction at f/billing/AGENTS is given to the assistant whenever it works on something under f/billing/. The file body is the instructions it follows.', + 'system', + 'md', + now() +) +ON CONFLICT (workspace_id, name) DO NOTHING; diff --git a/docs/auth-surface.md b/docs/auth-surface.md index a383b9267d..15f2538cc5 100644 --- a/docs/auth-surface.md +++ b/docs/auth-surface.md @@ -46,6 +46,12 @@ Symbols, not line numbers, are cited: they drift less. (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user token stored as a secret, never `$WM_TOKEN`. Token scopes cannot narrow superadmin routes. +- **Write access to a folder steers its readers' AI chat.** An `ai_instruction` resource's body is + delivered to the global chat of everyone who can read the resource, with no opt-in, whenever the + chat touches a path under its folder (`copilot/chat/folderInstructions.ts`); the chat then acts + with that reader's permissions, outside the folder too. `ai_skill` bodies are likewise in play + for every reader until turned off. The resource ACL is the only control, so granting write on a + folder grants that influence. - **A remote deploy token is a credential for another instance**, held per account and workspace (`remote_deploy_token`, encrypted under the workspace key, re-keyed by `set_encryption_key`). Its `email` references `password(email)` with `ON DELETE/UPDATE CASCADE`, so whatever deletes or diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 9803735571..c4a6efeabc 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -128,6 +128,11 @@ import type { WorkspaceMutationTarget } from './workspaceTools' import { resolveSessionAccess } from './global/sessionAccess' import type { SessionAccess } from './sessionCapabilities' import { filterSessionTools } from './global/sessionToolset' +import { + listFolderInstructions, + type FolderInstruction, + type FolderInstructionsDelivery +} from './folderInstructions' import { loadWorkspaceSkills, resolveGlobalPromptIdentity, @@ -1349,6 +1354,13 @@ export class AIChatManager implements ChatViewHost { globalSkills = $state([]) private globalSkillsRefreshId = 0 + // The `ai_instruction` resources readable in the operating workspace, loaded + // alongside skills. The prompt names their folders; the tool loop delivers each + // one's body the first time a call touches its folder, recording which call did. + private globalFolderInstructions: FolderInstruction[] = [] + private globalFolderInstructionsRefreshId = 0 + private folderInstructionsDeliveredBy = new Map() + // External MCP servers the user connected (resources of type `mcp`). Loaded // asynchronously alongside skills; the MCP tools are only registered when // this is non-empty, so a workspace with no connection pays no schema cost @@ -2428,6 +2440,7 @@ export class AIChatManager implements ChatViewHost { this.configureGlobalMode() void this.refreshGlobalIdentity() void this.refreshGlobalSkills() + void this.refreshGlobalFolderInstructions() void this.refreshMcpServers() } else if (mode === AIMode.APP) { const customPrompt = getCombinedCustomPrompt(mode) @@ -2482,6 +2495,7 @@ export class AIChatManager implements ChatViewHost { previewTools: this.isSessionChat, user: this.globalIdentity, skills: this.globalSkills, + folderInstructions: this.globalFolderInstructions, mcpServers: this.mcpServers, access: this.sessionAccess, sessionContext: this.sessionContextResolver?.(), @@ -2507,6 +2521,25 @@ export class AIChatManager implements ChatViewHost { } } + // Same shape as refreshGlobalSkills. + refreshGlobalFolderInstructions = async (workspace = this.operatingWorkspace ?? '') => { + const refreshId = ++this.globalFolderInstructionsRefreshId + let instructions: FolderInstruction[] = [] + try { + instructions = await listFolderInstructions(workspace) + } catch (e) { + console.error('Failed to load folder instructions', e) + } + if (refreshId !== this.globalFolderInstructionsRefreshId) { + return + } + this.globalFolderInstructions = + workspace === (this.operatingWorkspace ?? '') ? instructions : [] + if (this.mode === AIMode.GLOBAL) { + this.configureGlobalMode() + } + } + // Same shape as refreshGlobalSkills. An identity that resolves after the operating // workspace moved describes the workspace left behind, so it is dropped, not installed. refreshGlobalIdentity = async (workspace = this.operatingWorkspace ?? '') => { @@ -3583,14 +3616,16 @@ export class AIChatManager implements ChatViewHost { return false } } - // Session chats commit their workspace in beforeSend; the identity, skills, MCP - // servers and capabilities must all match the committed workspace before the system - // prompt is sent. Settling them here rather than mid-turn also keeps the prompt — the - // cached prefix of every iteration — stable for the whole request. + // Session chats commit their workspace in beforeSend; the identity, skills, folder + // instructions, MCP servers and capabilities must all match the committed workspace + // before the system prompt is sent. Settling them here rather than mid-turn also + // keeps the prompt — the cached prefix of every iteration — stable for the whole + // request. if (this.mode === AIMode.GLOBAL) { await Promise.all([ this.refreshGlobalIdentity(this.operatingWorkspace ?? ''), this.refreshGlobalSkills(this.operatingWorkspace ?? ''), + this.refreshGlobalFolderInstructions(this.operatingWorkspace ?? ''), this.refreshMcpServers(this.operatingWorkspace ?? ''), this.refreshSessionAccess(this.operatingWorkspace ?? '') ]) @@ -4050,7 +4085,11 @@ export class AIChatManager implements ChatViewHost { ? { onJobStarted: (job) => this.registerJob(job), onJobStatus: (jobId, update) => this.updateJob(jobId, update), - onJobDetached: (jobId) => this.markJobDetached(jobId) + onJobDetached: (jobId) => this.markJobDetached(jobId), + folderInstructions: { + list: () => this.globalFolderInstructions, + deliveredBy: this.folderInstructionsDeliveredBy + } } : {}), removeToolStatus: (id) => { diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index d5fee5a4d9..a469b3d35c 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -10,7 +10,8 @@ FileText, PanelRight, Lock, - ExternalLink + ExternalLink, + BookOpen } from 'lucide-svelte' import { base } from '$lib/base' import { @@ -171,6 +172,18 @@ {message.toolName} {/if}
    +{:else if message.heldForFolderInstructions} + +
    + + + {message.content} + + {#if message.toolName} + {message.toolName} held until read + {/if} +
    {:else if isRunCard} {:else if isDiffCard} diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 122425e264..ab69d0b1a3 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -446,7 +446,8 @@ export async function parseAnthropicCompletion( toolCall, helpers, toolCallbacks: callbacks, - workspace: options?.workspace + workspace: options?.workspace, + messages }) messages.push(messageToAdd) addedMessages.push(messageToAdd) diff --git a/frontend/src/lib/components/copilot/chat/folderInstructions.ts b/frontend/src/lib/components/copilot/chat/folderInstructions.ts new file mode 100644 index 0000000000..c07c85cbbd --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/folderInstructions.ts @@ -0,0 +1,220 @@ +import { ResourceService } from '$lib/gen' +import { truncateForPrompt } from './skills/skillResources' + +/** + * Folder instructions are resources of this type: a markdown file resource + * (`format_extension = 'md'`) whose `value.content` the assistant follows for + * every item under the directory the resource lives in, like an AGENTS.md. + */ +export const FOLDER_INSTRUCTIONS_RESOURCE_TYPE = 'ai_instruction' + +/** Bytes of one instruction body a tool result may carry. */ +export const MAX_FOLDER_INSTRUCTIONS_LENGTH = 32 * 1024 +/** Bytes all the bodies one tool result carries may add up to, so nested folders + * cannot stack several full-size bodies onto a single result. */ +export const MAX_FOLDER_INSTRUCTIONS_TOTAL = 48 * 1024 + +/** Scopes the system prompt names. Past this the list is cut, not the delivery: + * a scope left out still has its instructions delivered on first touch. */ +export const MAX_LISTED_FOLDER_INSTRUCTIONS = 50 + +export type FolderInstruction = { + /** The `ai_instruction` resource. */ + path: string + /** The directory it covers, with a trailing `/`: `f/billing/AGENTS` covers `f/billing/`. */ + scope: string +} + +export function folderInstructionScope(path: string): string { + return path.slice(0, path.lastIndexOf('/') + 1) +} + +const PAGE_SIZE = 100 +/** Guard against a paging bug looping forever, not a product cap. */ +const MAX_PAGES = 20 + +/** Every folder instruction the user can read in the workspace. Read access is the + * resource's own ACL, so a folder's instructions reach exactly who can see them. */ +export async function listFolderInstructions(workspace: string): Promise { + if (!workspace) return [] + const rows: FolderInstruction[] = [] + for (let page = 1; page <= MAX_PAGES; page++) { + const resources = await ResourceService.listResource({ + workspace, + resourceType: FOLDER_INSTRUCTIONS_RESOURCE_TYPE, + page, + perPage: PAGE_SIZE + }) + rows.push(...resources.map((r) => ({ path: r.path, scope: folderInstructionScope(r.path) }))) + if (resources.length < PAGE_SIZE) break + } + return rows +} + +/** Workspace paths a tool call names: every string argument called `path` or + * `*_path` that is a `u/`, `f/` or `g/` path, nested ones included — a trigger's target + * sits in `config.path`, and missing it would let that write through unheld. */ +export function workspacePathsInArgs(args: unknown, depth = 0): string[] { + if (!args || typeof args !== 'object' || depth > 4) return [] + if (Array.isArray(args)) return args.flatMap((v) => workspacePathsInArgs(v, depth + 1)) + return Object.entries(args as Record).flatMap(([key, value]) => + typeof value === 'string' + ? (key === 'path' || key.endsWith('_path')) && /^[ufg]\/[^/]+/.test(value) + ? [value] + : [] + : workspacePathsInArgs(value, depth + 1) + ) +} + +/** The instructions covering any of `paths`, outermost scope first, so a nested + * folder's instructions come last and read as the more specific ones. A path naming + * the folder itself (`f/billing`, as a pipeline's is) is covered too. */ +export function instructionsCovering( + instructions: readonly FolderInstruction[], + paths: readonly string[] +): FolderInstruction[] { + return instructions + .filter((i) => paths.some((p) => `${p}/`.startsWith(i.scope))) + .sort((a, b) => a.scope.length - b.scope.length || a.path.localeCompare(b.path)) +} + +/** What a chat hands the tool loop: the instructions in play, and which tool call + * delivered which of them. + * + * A delivery counts only while its tool message is still in the conversation and + * carries the text: the id is recorded because a script quoting the tag in a result + * must not pass for a delivery, and nothing is marked on the message object because + * the completions path sends that to the provider verbatim. So compaction or a + * restored chat gets the instructions again. */ +export type FolderInstructionsContext = { + list: () => readonly FolderInstruction[] + deliveredBy: Map +} + +/** Keyed by workspace too: the chat keeps its conversation across a switch of + * operating workspace, and the same path there can hold different instructions. */ +export type FolderInstructionsDelivery = { workspace: string; paths: readonly string[] } + +/** Instructions delivered by the tool results in `messages`, split by whether the + * model has read them. Results after the latest assistant message answer the batch + * still being processed: the model has not seen those yet, so a delivery there + * must not let a change later in the same batch through. */ +function deliveredIn( + ctx: FolderInstructionsContext, + messages: readonly unknown[], + workspace: string +): { read: Set; thisBatch: Set } { + const read = new Set() + const thisBatch = new Set() + let lastAssistant = -1 + messages.forEach((m, i) => { + if ((m as { role?: unknown } | undefined)?.role === 'assistant') lastAssistant = i + }) + messages.forEach((m, i) => { + const { tool_call_id: id, content } = (m ?? {}) as { tool_call_id?: unknown; content?: unknown } + if (typeof id !== 'string' || typeof content !== 'string') return + const delivery = ctx.deliveredBy.get(id) + if (delivery?.workspace !== workspace) return + for (const p of delivery.paths) { + // The id alone is not enough: a call stopped mid-run is answered with a + // placeholder under the same id, which carries no instructions. + if (content.includes(openingTag(p))) (i < lastAssistant ? read : thisBatch).add(p) + } + }) + return { read, thisBatch } +} + +/** The instructions covering the paths a tool call names that the model has not + * read yet, as the text to hand it, or undefined when there are none. `paths` are + * the bodies the text carries; one already carried earlier in the same batch is + * pointed to rather than repeated. A body that no longer reads (deleted or moved + * since the listing) is left out rather than failing the call it rides on. */ +export async function pendingFolderInstructions( + ctx: FolderInstructionsContext | undefined, + messages: readonly unknown[], + args: unknown, + workspace: string +): Promise<{ paths: string[]; text: string } | undefined> { + if (!ctx || !workspace) return undefined + const paths = workspacePathsInArgs(args) + if (paths.length === 0) return undefined + const { read, thisBatch } = deliveredIn(ctx, messages, workspace) + const pending = instructionsCovering(ctx.list(), paths).filter((i) => !read.has(i.path)) + const inBatch = pending.filter((i) => thisBatch.has(i.path)) + const blocks = ( + await Promise.all( + pending + .filter((i) => !thisBatch.has(i.path)) + .map(async (instruction) => { + try { + return { + instruction, + body: await readFolderInstructionBody(workspace, instruction.path) + } + } catch (e) { + console.error(`Failed to read folder instructions ${instruction.path}`, e) + return undefined + } + }) + ) + ).filter((b) => b !== undefined) + if (blocks.length === 0 && inBatch.length === 0) return undefined + const pointers = inBatch.map( + (i) => + `The instructions for \`${i.scope}\` (${i.path}) are in an earlier result of this same batch of tool calls.` + ) + return { + paths: blocks.map((b) => b.instruction.path), + text: [formatFolderInstructions(blocks), ...pointers].filter(Boolean).join('\n\n') + } +} + +export async function readFolderInstructionBody(workspace: string, path: string): Promise { + const value = (await ResourceService.getResourceValue({ workspace, path })) as + | { content?: unknown } + | undefined + if (typeof value?.content !== 'string') { + throw new Error(`resource ${path} has no string "content"`) + } + return value.content +} + +function openingTag(path: string): string { + return ` { + const cut = truncateForPrompt(body, Math.min(MAX_FOLDER_INSTRUCTIONS_LENGTH, remaining)) + remaining = Math.max(0, remaining - new TextEncoder().encode(cut).byteLength) + return cut + }) + bodies.reverse() + return blocks + .map( + ({ instruction }, i) => + `${openingTag(instruction.path)} scope="${instruction.scope}">\n${bodies[i]}\n` + ) + .join('\n\n') +} + +/** The system prompt section naming the scopes that carry instructions. */ +export function getFolderInstructionsPromptSection( + instructions: readonly FolderInstruction[] +): string { + if (instructions.length === 0) return '' + const scopes = [...new Set(instructions.map((i) => i.scope))].sort() + const listed = scopes.slice(0, MAX_LISTED_FOLDER_INSTRUCTIONS) + const more = scopes.length > listed.length ? ` (and ${scopes.length - listed.length} more)` : '' + return `\n\nFOLDER INSTRUCTIONS: +These folders carry instructions (ai_instruction resources) for every item under them: ${listed.map((s) => `\`${s}\``).join(', ')}${more}. +- The first time a tool call names a path under one of them, its instructions are delivered in a block of the tool result. A call that would change something is held back until you have them: read them, then make the call again. +- Follow them for all work under that folder, together with any WORKSPACE INSTRUCTIONS and USER INSTRUCTIONS above: they add to those, they do not replace them. +- Only where two of them conflict on the same point, the more specific one wins: a nested folder's instructions over its parent folder's, a folder's over the WORKSPACE INSTRUCTIONS, and both over the USER INSTRUCTIONS, which are personal preferences. The user's explicit requests in this chat come before all of them.` +} diff --git a/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts b/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts index 58e45c6d08..a291e93ec7 100644 --- a/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts +++ b/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts @@ -13,6 +13,7 @@ import { } from './core' import { createMcpTools } from './mcpTools' import { getPipelinePromptSection, pipelineTools, type PipelineContext } from '../pipeline/core' +import { getFolderInstructionsPromptSection, type FolderInstruction } from '../folderInstructions' // Derived, not retyped: an option added to prepareGlobalSystemMessage is reachable // here at once. Hand-listing the four would compile fine while leaving the new one @@ -21,6 +22,7 @@ import { getPipelinePromptSection, pipelineTools, type PipelineContext } from '. export type GlobalAssemblyOpts = NonNullable[1]> & { sessionContext?: SessionPromptContext pipelineContext?: PipelineContext + folderInstructions?: readonly FolderInstruction[] } export function assembleGlobalSystemMessage( @@ -28,6 +30,7 @@ export function assembleGlobalSystemMessage( opts: GlobalAssemblyOpts ): ChatCompletionSystemMessageParam { const systemMessage = prepareGlobalSystemMessage(instructions, opts) + systemMessage.content += getFolderInstructionsPromptSection(opts.folderInstructions ?? []) if (opts.sessionContext) { systemMessage.content += getSessionContextPromptSection(opts.sessionContext, opts.access) } diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 0e460f1921..31d3b379ac 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -601,7 +601,8 @@ export async function parseOpenAIResponsesCompletion( toolCall, helpers, toolCallbacks: callbacks, - workspace: options?.workspace + workspace: options?.workspace, + messages }) messages.push(messageToAdd) addedMessages.push(messageToAdd) diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index fb61f3a912..4574df7355 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -54,7 +54,10 @@ vi.mock('$lib/gen', () => ({ AzureTriggerService: { createAzureTrigger: vi.fn() }, AmqpTriggerService: { createAmqpTrigger: vi.fn() }, EmailTriggerService: { createEmailTrigger: vi.fn() }, - SettingService: { getGlobal: vi.fn() } + SettingService: { getGlobal: vi.fn() }, + ResourceService: { + getResourceValue: vi.fn(async ({ path }: { path: string }) => ({ content: `body of ${path}` })) + } })) vi.mock('$lib/utils', () => ({ @@ -205,6 +208,106 @@ describe('buildContextString', () => { }) describe('processToolCall', () => { + it('delivers folder instructions once, holding back a change until they are read', async () => { + const { createToolDef, processToolCall } = await import('./shared') + const read = vi.fn(async () => 'read ok') + const write = vi.fn(async () => 'write ok') + const tools = [ + { def: createToolDef(z.object({}), 'read_item', 'Read'), planModeSafe: true, fn: read }, + { def: createToolDef(z.object({}), 'write_item', 'Write'), fn: write } + ] + const folderInstructions = { + list: () => [ + { path: 'f/billing/AGENTS', scope: 'f/billing/' }, + { path: 'f/billing/eu/AGENTS', scope: 'f/billing/eu/' }, + { path: 'f/other/AGENTS', scope: 'f/other/' }, + { path: 'g/ops/AGENTS', scope: 'g/ops/' } + ], + deliveredBy: new Map() + } + const messages: any[] = [] + const call = async ( + id: string, + name: string, + path: string | object, + workspace = 'test-workspace' + ) => { + const message = await processToolCall({ + tools, + toolCall: { + id, + type: 'function', + function: { + name, + arguments: JSON.stringify(typeof path === 'string' ? { path } : path) + } + }, + helpers: {}, + workspace, + messages, + toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn(), folderInstructions } + }) + messages.push(message) + return message.content as string + } + const turn = () => messages.push({ role: 'assistant', content: '' }) + + turn() + const readResult = await call('c1', 'read_item', 'f/billing/eu/invoice') + expect(read).toHaveBeenCalledTimes(1) + expect(readResult.startsWith('read ok')).toBe(true) + // Outermost first, so the nested folder's instructions read as the specific ones. + expect(readResult.indexOf('body of f/billing/AGENTS')).toBeLessThan( + readResult.indexOf('body of f/billing/eu/AGENTS') + ) + // Same batch: the model has not read c1's result yet, so the write still waits, + // pointed at that result rather than handed the bodies twice. + const sameBatch = await call('c2', 'write_item', 'f/billing/eu/invoice') + expect(write).not.toHaveBeenCalled() + expect(sameBatch).not.toContain('body of f/billing/AGENTS') + expect(sameBatch).toContain('earlier result of this same batch') + + turn() + expect(await call('c3', 'write_item', 'f/billing/eu/invoice')).toBe('write ok') + + const held = await call('c4', 'write_item', 'f/other/x') + expect(write).toHaveBeenCalledTimes(1) + expect(held).toContain('body of f/other/AGENTS') + expect(held).not.toContain('body of f/billing/AGENTS') + + turn() + expect(await call('c5', 'write_item', 'f/other/x')).toBe('write ok') + + expect(await call('g1', 'read_item', 'g/ops/runbook')).toContain('body of g/ops/AGENTS') + + // The same path in another workspace is another resource, delivered afresh. + expect(await call('w1', 'read_item', 'f/other/x', 'other-workspace')).toContain( + 'body of f/other/AGENTS' + ) + + // A delivery lost from the conversation (compaction) is made again. + messages.splice(0) + turn() + expect(await call('c6', 'read_item', 'f/billing/y')).toContain('body of f/billing/AGENTS') + + // A trigger names its target inside its config. + messages.splice(0) + turn() + const nested = await call('t1', 'write_item', { kind: 'http', config: { path: 'f/other/t' } }) + expect(nested).toContain('body of f/other/AGENTS') + expect(write).toHaveBeenCalledTimes(2) + + // A call stopped mid-run is answered with a placeholder under its id: no delivery. + messages.splice(0) + folderInstructions.deliveredBy.set('stopped', { + workspace: 'test-workspace', + paths: ['f/billing/AGENTS'] + }) + messages.push({ role: 'tool', tool_call_id: 'stopped', content: 'Interrupted' }) + turn() + expect(await call('c7', 'read_item', 'f/billing/y')).toContain('body of f/billing/AGENTS') + }) + it('returns pre-confirmation validation errors without asking for confirmation', async () => { const { createToolDef, processToolCall } = await import('./shared') const error = 'the script needs to be deployed before doing this action' diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 5e8bdb8a4b..acb3f95d0d 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -60,6 +60,7 @@ import { forLater } from '$lib/forLater' import { scriptLangToEditorLang } from '$lib/scripts' import { getCurrentModel } from '$lib/aiStore' import { type editor as meditor } from 'monaco-editor' +import { pendingFolderInstructions, type FolderInstructionsContext } from './folderInstructions' // Prettify function for code arguments - extracts and formats code from JSON function prettifyCodeArguments(content: string): string { @@ -743,6 +744,8 @@ export type ToolDisplayMessage = { /** Refused by the plan-mode gate. Renders as its own lean row rather than a tool * error, so the transcript says the mode stopped it and not that the call failed. */ blockedByPlanMode?: boolean + /** Held back until the model has read the folder instructions it was handed. */ + heldForFolderInstructions?: boolean /** The user declined: the reject button, a Stop, or a posture switch. Set only there, so * a decision is distinguishable from every other way a call errors. */ declinedByUser?: boolean @@ -975,20 +978,29 @@ function stringifyErrorBody(body: unknown): string { * (tab closed while a tool polls) logs nothing, so the statuses sum to the calls that * finished, not to the calls made. */ -type ToolCallStatus = 'ok' | 'error' | 'declined' | 'rejected' | 'blocked_plan_mode' +type ToolCallStatus = + | 'ok' + | 'error' + | 'declined' + | 'rejected' + | 'blocked_plan_mode' + | 'held_for_instructions' export async function processToolCall({ tools, toolCall, helpers, toolCallbacks, - workspace + workspace, + messages = [] }: { tools: Tool[] toolCall: ChatCompletionMessageFunctionToolCall helpers: T toolCallbacks: ToolCallbacks workspace?: string + /** The conversation so far, for which folder instructions it already carries. */ + messages?: readonly ChatCompletionMessageParam[] }): Promise { const tool = tools.find((t) => t.def.function.name === toolCall.function.name) const workspaceId = workspace ?? get(workspaceStore) ?? '' @@ -1072,6 +1084,43 @@ export async function processToolCall({ } } + // After the gates that refuse the call, so a refused call does not use up the + // delivery. A call that changes something is held back until the model has read + // the instructions: arriving with its result, they would be too late to shape it. + const folderInstructions = tool + ? await pendingFolderInstructions( + toolCallbacks.folderInstructions, + messages, + args, + workspaceId + ) + : undefined + if (folderInstructions?.paths.length) { + toolCallbacks.folderInstructions?.deliveredBy.set(toolCall.id, { + workspace: workspaceId, + paths: folderInstructions.paths + }) + } + if (folderInstructions && tool?.planModeSafe !== true) { + logToolOutcome('held_for_instructions') + toolCallbacks.setToolStatus(toolCall.id, { + content: 'Read folder instructions', + heldForFolderInstructions: true, + parameters: args, + isLoading: false, + isQueued: false, + isStreamingArguments: false, + needsConfirmation: false, + showDetails: tool?.showDetails, + autoCollapseDetails: tool?.autoCollapseDetails + }) + return { + role: 'tool' as const, + tool_call_id: toolCall.id, + content: `Not run: this call touches a folder whose instructions you had not read yet. Follow them, then make the call again (adjusted if they require it).\n\n${folderInstructions.text}` + } + } + const requiresConfirmation = tool?.requiresConfirmation === true // By name: skipping the wait is itself an answer on the user's behalf, and one tool // must not be answered for. @@ -1176,7 +1225,7 @@ export async function processToolCall({ const toAdd = { role: 'tool' as const, tool_call_id: toolCall.id, - content: result + content: folderInstructions ? `${result}\n\n${folderInstructions.text}` : result } return toAdd } catch (err) { @@ -1454,6 +1503,9 @@ export interface ToolCallbacks { attachToolImage?: (toolId: string, image: AttachedImage) => void /** Drain every image buffered this batch (insertion order), clearing the buffer. */ takePendingToolImages?: () => AttachedImage[] + /** Wired only by the global/sessions chat: delivers the folder instructions + * covering the paths a tool call names, once per conversation. */ + folderInstructions?: FolderInstructionsContext } export function createToolDef( diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index b55aa06c92..655a872f96 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1433,7 +1433,8 @@ export async function parseOpenAICompletion( toolCall, helpers, toolCallbacks: callbacks, - workspace: options?.workspace + workspace: options?.workspace, + messages }) messages.push(messageToAdd) addedMessages.push(messageToAdd) From c232ea43b4bfe1ed295dd2f71cc6cefa47772bed Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 24 Sep 2026 14:53:36 +0200 Subject: [PATCH 03/91] fix: limit compare page guidance to fork deploys (#11300) * fix: limit compare page guidance to fork deploys * fix: make open_page compare fork-only and drop stale api catalog bullet * fix: always open the compare page in fork mode from open_page * fix: drop the ignored mode argument from open_page --- ai_evals/cases/global.yaml | 32 ++++++--- .../copilot/chat/AIChatManager.svelte.ts | 1 - .../copilot/chat/global/core.test.ts | 38 +++------- .../components/copilot/chat/global/core.ts | 72 +++++-------------- 4 files changed, 47 insertions(+), 96 deletions(-) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index b17d37cb29..96aa3ab983 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1112,9 +1112,9 @@ - opens the Kafka triggers page - does not write, deploy, or delete anything -- id: global-openpage7-compare-review +- id: global-draft-does-not-open-compare prompt: |- - Create a TypeScript script draft at f/evals/global/compare_review_demo that returns the string "ok" (no need to test it), then open the review page so I can look over the pending change and deploy it myself. + Create a TypeScript script draft at f/evals/global/compare_review_demo that returns the string "ok" (no need to test it). I want to look over the change before it gets deployed. runtime: maxTurns: 8 validate: @@ -1122,6 +1122,24 @@ toolExpect: requiredToolsUsed: - write_script + forbiddenToolsUsed: + - open_page + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - creates the script draft without opening any workspace page + - does not deploy or delete anything + +- id: global-openpage7-compare-fork-deploy + prompt: |- + I'm done with my changes in this fork and want to ship them to the parent workspace myself. Take me there. + runtime: + maxTurns: 6 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: - open_page forbiddenToolsUsed: - deploy_workspace_item @@ -1131,17 +1149,9 @@ field: page stringIncludesAnyOf: - compare - # The eval chat is untracked (no modified-items mask), so the model must scope - # the review by passing the item it changed explicitly — an omitted mask would - # preselect every pending change in the workspace. - - tool: open_page - field: items - stringIncludesAnyOf: - - f/evals/global/compare_review_demo skipJudge: true judgeChecklist: - - creates the script draft, then opens the Compare & Deploy review page instead of deploying itself - - preselects only the created script on the review page + - opens Compare & Deploy only because the user explicitly wants to deploy the fork into its parent - does not deploy or delete anything - id: global-openpage8-runs-label-and-worker diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index c4a6efeabc..6e51f80601 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -2471,7 +2471,6 @@ export class AIChatManager implements ChatViewHost { : {}), testActiveFlow: async (storagePath: string, args?: Record, memoryId?: string) => this.flowEditorFor(storagePath)?.testFlow(args, memoryId), - getModifiedItems: () => (this.modifiedItems ? [...this.modifiedItems] : undefined), attachedFiles: this.attachedFiles, getUserInstructions: () => getUserCustomPrompts()[AIMode.GLOBAL] ?? '', setUserInstructions: (instructions: string) => { diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 3c4a0c80ed..820c593514 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -7616,36 +7616,14 @@ describe('buildOpenPageUrl runs filters', () => { }) }) -describe('buildOpenPageUrl compare selection', () => { - const itemsOf = (url: string) => new URL(url, 'http://x').searchParams.get('items') - - it('explicit items win over the chat mask', () => { - const url = buildOpenPageUrl( - 'compare', - { page: 'compare', items: ['script:f/a/b'] }, - { workspaceId: 'ws', chatItems: ['flow:f/c/d'] } - ) - expect(itemsOf(url)).toBe('script:f/a/b') - }) - - it('omitted items fall back to the chat-modified mask', () => { - const url = buildOpenPageUrl( - 'compare', - { page: 'compare' }, - { workspaceId: 'ws', chatItems: ['flow:f/c/d', 'script:f/a/b'] } - ) - expect(itemsOf(url)).toBe('flow:f/c/d,script:f/a/b') - }) - - it('an empty or absent mask yields no items param (page select-all default)', () => { - expect( - itemsOf( - buildOpenPageUrl('compare', { page: 'compare' }, { workspaceId: 'ws', chatItems: [] }) - ) - ).toBeNull() - expect( - itemsOf(buildOpenPageUrl('compare', { page: 'compare' }, { workspaceId: 'ws' })) - ).toBeNull() +describe('buildOpenPageUrl compare', () => { + // Without an explicit mode the page auto-picks, which is the draft view outside a fork. + it('always opens the fork comparison', () => { + const params = new URL( + buildOpenPageUrl('compare', { page: 'compare' }, { workspaceId: 'ws' }), + 'http://x' + ).searchParams + expect(params.get('mode')).toBe('fork') }) }) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 9a5d1f43f1..b646b1283d 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1438,11 +1438,7 @@ ${pipelineBullet}` - Whenever you ask the user to perform a manual step in the UI — fill in a resource's credentials, set a secret variable's value, adjust a schedule or setting — call open_page in the same message, targeted at that item (pass open with its path to land in its editor, or the page's filters otherwise). Never just describe where to click.${when( canWriteDraft, ` -- When the user is happy with the changes and wants to review or deploy them, use open_page with page "compare" — it opens the Compare & Deploy review page.${ - previewTools - ? ' By default it preselects the items this chat modified; pass items (":" entries) to control the selection' - : ' Pass items (":" entries naming the items you changed) so the review is scoped to them — omitting items preselects every pending change in the workspace' - }, or mode ("draft" or "fork") to force which comparison is shown. Prefer offering this review page over calling deploy_workspace_item directly when several items changed.` +- Do not offer or open the Compare & Deploy page for normal draft review. Only use open_page with page "compare" when the user explicitly asks to deploy a forked workspace's changes to its parent workspace.` )}${when( canRunPreview, ` @@ -2803,17 +2799,6 @@ function allowsAllWorkspacesRuns(workspaceId: string | undefined = get(workspace return (!!get(superadmin) || !!get(devopsRole)) && workspaceId === 'admins' } -// The advertised `items` description must match this chat's surface: only chats that -// track their modified items (AI sessions) can honor "omitted = this chat's edits" — -// on an untracked chat (the global side panel) an omitted mask falls through to the -// page's select-all default, so the model is told to pass the items explicitly there. -const COMPARE_ITEMS_DESCRIPTIONS = { - tracked: - "Compare: preselect exactly these changed items, each as ':' where kind is script, flow, raw_app, app, resource, variable, or a trigger kind like trigger_schedule / trigger_http (e.g. 'script:f/foo/bar'). Omit to preselect the items modified in this chat (everything when this chat modified nothing).", - untracked: - "Compare: preselect exactly these changed items, each as ':' where kind is script, flow, raw_app, app, resource, variable, or a trigger kind like trigger_schedule / trigger_http (e.g. 'script:f/foo/bar'). If omitted, the page preselects EVERY pending change in the workspace, not just this chat's — when you changed specific items, pass them so the review is scoped to them." -} as const - // The Runs filters the page accepts several values for, all encoded in one param: the // value is a comma-separated list, and for the negatable ones a leading `!` excludes // instead (the page rejects a list mixing included and excluded values). @@ -2979,13 +2964,13 @@ const openPageFullSchema = z.object({ .enum([...WORKSPACE_SETTINGS_TABS] as [string, ...string[]]) .optional() .describe('Workspace settings: which settings tab to open'), - mode: z - .enum(['draft', 'fork']) + items: z + .array(z.string()) + .min(1) .optional() .describe( - "Compare: which comparison to show — 'draft' (deployed items vs their pending drafts) or 'fork' (this forked workspace vs its parent). Omit to auto-pick: the view containing the preselected items (draft whenever any of them is a pending draft); with nothing preselected, fork on a forked workspace and draft otherwise." + "Compare: preselect exactly these fork changes, each as ':' where kind is script, flow, raw_app, app, resource, variable, or a trigger kind like trigger_schedule / trigger_http (e.g. 'script:f/foo/bar'). Omit to use the page's default selection; pass items only when the user asked to deploy specific ones." ), - items: z.array(z.string()).min(1).optional().describe(COMPARE_ITEMS_DESCRIPTIONS.tracked), new_tab: z .boolean() .optional() @@ -3029,7 +3014,6 @@ const OPEN_PAGE_FIELD_PAGES: Record = { operation: ['audit_logs'], resource: ['audit_logs'], tab: ['workspace_settings'], - mode: ['compare'], items: ['compare'] } @@ -3040,7 +3024,6 @@ const OPEN_PAGE_FIELD_PAGES: Record = { function buildOpenPageDefSchema( pages: readonly OpenPageName[], triggerKinds: readonly PageTriggerKind[], - chatEditsTracked: boolean, allWorkspacesRuns: boolean, roleUnverified = false ): z.ZodTypeAny { @@ -3070,25 +3053,18 @@ function buildOpenPageDefSchema( .enum([...triggerKinds] as [string, ...string[]]) .optional() .describe('Triggers: which trigger kind page to open') - : field === 'items' - ? z - .array(z.string()) - .min(1) - .optional() - .describe(COMPARE_ITEMS_DESCRIPTIONS[chatEditsTracked ? 'tracked' : 'untracked']) - : full[field] + : full[field] } shape.new_tab = full.new_tab return z.object(shape) } const OPEN_PAGE_DESCRIPTION = - 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy review page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its editor opens directly). Use page "compare" when the user wants to review and deploy pending changes (the items field controls which changes are preselected). This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' + 'Open a Windmill page with filters applied — Runs, Schedules, Variables, Resources, Assets, Audit logs, Folders, Groups, Triggers (by kind), Workspace settings (on a specific tab), or the Compare & Deploy page. Inside an AI session it opens as a tab in the side-panel preview next to the chat; elsewhere it offers a clickable link. Use after surfacing something the user likely wants to inspect (e.g. "show me the failed runs of X", "open the schedule for Y", "open the git sync settings", "open the kafka triggers"), and ALWAYS when asking the user to perform a manual step themselves (fill in a resource\'s credentials, set a variable\'s value — pass open with the item path so its editor opens directly). Never offer page "compare" for draft review. Use it only when the user explicitly asks to deploy a forked workspace into its parent; it always opens the fork-vs-parent comparison. This is the only way to show one of these pages in the session preview — open_preview only handles editable items (scripts, flows, raw apps, pipelines). Only pages listed for this user are available; do not offer others.' -// Non-arg inputs the URL builder needs: the chat's operating workspace (the compare -// page cannot fall back to its own store default inside a session preview) and the -// live modified-items mask backing the compare page's default preselection. -type OpenPageUrlCtx = { workspaceId: string; chatItems?: readonly string[] } +// Non-arg input the URL builder needs: the chat's operating workspace (the compare +// page cannot fall back to its own store default inside a session preview). +type OpenPageUrlCtx = { workspaceId: string } // The Runs page reads its two absolute bounds as `new Date(param)` and drops whatever // doesn't parse, so normalize to ISO here rather than passing a stamp the page will @@ -3241,14 +3217,11 @@ export function buildOpenPageUrl(page: OpenPageName, a: OpenPageArgs, ctx: OpenP case 'workspace_settings': return buildWorkspaceSettingsUrl({ tab: a.tab }) case 'compare': - // Explicit `items` wins; otherwise preselect this chat's modified items. An - // empty mask (chat modified nothing) passes no items so the page keeps its - // select-all default instead of preselecting nothing. - return buildCompareUrl({ - workspace_id: ctx.workspaceId, - mode: a.mode, - items: a.items ?? (ctx.chatItems?.length ? ctx.chatItems : undefined) - }) + // Always fork: an omitted mode lets the page auto-pick the draft view. No + // chat-modified fallback for `items` either: a session's edits are usually + // undeployed drafts, which the fork comparison leaves out, so masking by them + // would open "deploy to parent" with nothing selected. + return buildCompareUrl({ workspace_id: ctx.workspaceId, mode: 'fork', items: a.items }) } } @@ -3277,13 +3250,12 @@ function summarizeOpenPage(url: string, page: OpenPageName): string { export const openPageTool: SessionTool<{}> = { requires: NONE, - // The initial def assumes an untracked chat and no resolved role; schemaFor below - // rebuilds it with the caller's real surface before each iteration. + // The initial def assumes no resolved role; schemaFor below rebuilds it with the + // caller's real surface before each iteration. def: createToolDef( buildOpenPageDefSchema( restrictedOpenPages(get(workspaceStore)), allowedTriggerKinds(), - false, allowsAllWorkspacesRuns() ), 'open_page', @@ -3302,7 +3274,6 @@ export const openPageTool: SessionTool<{}> = { buildOpenPageDefSchema( access.pages, allowedTriggerKinds(), - (helpers as GlobalToolHelpers | undefined)?.getModifiedItems?.() !== undefined, allowsAllWorkspacesRuns(operatingWorkspaceFromHelpers(helpers)), access.roleUnverified ), @@ -3348,10 +3319,7 @@ export const openPageTool: SessionTool<{}> = { if (!urlWorkspace) { return 'Error: no workspace is selected, so no page can be opened.' } - const url = buildOpenPageUrl(page, parsed, { - workspaceId: urlWorkspace, - chatItems: (ctx.helpers as GlobalToolHelpers | undefined)?.getModifiedItems?.() - }) + const url = buildOpenPageUrl(page, parsed, { workspaceId: urlWorkspace }) const pageLabel = OPEN_PAGE_LABELS[page] const summary = summarizeOpenPage(url, page) @@ -4709,10 +4677,6 @@ export type GlobalToolHelpers = SessionToolHelpers & { // Wired only for session chats (see AIChatManager): the artifact tools are session-gated. artifacts?: SessionArtifactsStore getChatId?: () => string | undefined - // Live snapshot of the items this chat modified (`kind:path` mask keys, see - // modifiedItemsMask.ts); undefined when the chat doesn't track them (the global - // side-panel chat). Backs open_page's compare-page default preselection. - getModifiedItems?: () => string[] | undefined openArtifact?: (artifactId: string, name: string, version?: ArtifactVersionTarget) => void } From 9b46b21f517a67867912092c619675d91a197313 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 24 Sep 2026 14:54:13 +0200 Subject: [PATCH 04/91] feat(cli): include extra paths in codebase digest for monorepo imports (#11327) * feat(cli): hash codebase extra_digest_paths and warn on uncovered bundle inputs Co-Authored-By: Claude Opus 5.5 * fix(cli): cache codebase digest only once complete, skip entry in warning Co-Authored-By: Claude Opus 5.5 * fix(cli): accept string extra_digest_paths, always print uncovered-input warning Co-Authored-By: Claude Opus 5.5 * fix(cli): clarify extra_digest_paths resolution and missing-path error Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cli/src/commands/init/template.ts | 2 + cli/src/commands/script/script.ts | 17 ++++++++- cli/src/core/conf.ts | 1 + cli/src/utils/codebase.ts | 53 +++++++++++++++++++++++++-- cli/test/codebase_digest_unit.test.ts | 48 ++++++++++++++++++++++++ cli/wmill.schema.json | 7 ++++ 6 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 cli/test/codebase_digest_unit.test.ts diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts index 9032cee397..8a433d69d4 100644 --- a/cli/src/commands/init/template.ts +++ b/cli/src/commands/init/template.ts @@ -140,6 +140,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [ type: "object", properties: { relative_path: { type: "string", description: "Path to the codebase directory" }, + extra_digest_paths: { type: "array", items: { type: "string" }, description: "Paths outside relative_path that scripts import, relative to wmill.yaml; hashed recursively" }, includes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to include in bundle" }, excludes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to exclude from bundle" }, format: { type: "string", enum: ["cjs", "esm"], description: "Bundle output format" }, @@ -159,6 +160,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [ example: [ "# codebases:", '# - relative_path: ./shared # path to the codebase', + '# # extra_digest_paths: ["../packages/utils/src"] # imported paths outside relative_path, hashed recursively (skip node_modules/dist)', '# includes: ["**/*.ts"] # files to include in bundle', '# excludes: ["node_modules/**"] # files to exclude', '# format: esm # bundle format: "cjs" or "esm"', diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index fe193e5d85..06db3db1bf 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -61,7 +61,11 @@ import { readConfigFile, readEffectiveSyncBehavior, } from "../../core/conf.ts"; -import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; +import { + SyncCodebase, + listSyncCodebases, + uncoveredBundleInputs, +} from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; import fs from "node:fs"; import { createTarBlob, type TarEntry } from "../../utils/tar.ts"; @@ -446,8 +450,19 @@ export async function handleFile( target: format == "cjs" ? "node20.15.1" : "esnext", banner: codebase.banner, // ...(codebase.banner != null && { banner: codebase.banner }), + metafile: true, }); const endTime = performance.now(); + const uncovered = uncoveredBundleInputs( + codebase, + path, + Object.keys(out.metafile?.inputs ?? {}) + ); + if (uncovered.length > 0) { + log.warnAlways( + `${path} bundles files outside codebase ${codebase.relative_path}; edits to them won't be detected as changes unless listed in extra_digest_paths: ${uncovered.join(", ")}` + ); + } bundleContent = out.outputFiles[0].text; outputFiles = out.outputFiles ?? []; if (outputFiles.length == 0) { diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 0c746d2261..7c6eb13274 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -115,6 +115,7 @@ export interface SyncOptions { export interface Codebase { relative_path: string; + extra_digest_paths?: string[]; includes?: string[]; excludes?: string[]; assets?: { diff --git a/cli/src/utils/codebase.ts b/cli/src/utils/codebase.ts index e665d060b2..2eb63caeec 100644 --- a/cli/src/utils/codebase.ts +++ b/cli/src/utils/codebase.ts @@ -1,6 +1,44 @@ import { Codebase, SyncOptions } from "../core/conf.ts"; import * as log from "../core/log.ts"; -import { digestDir } from "./utils.ts"; +import { digestDir, generateHash, generateHashFromBuffer } from "./utils.ts"; +import { readFile, stat } from "node:fs/promises"; +import path from "node:path"; + +// A bare string is accepted, like `includes`/`excludes` in findCodebase. +function extraDigestPaths(codebase: Codebase): string[] { + return ([] as string[]).concat(codebase.extra_digest_paths ?? []); +} + +async function digestPath(p: string): Promise { + let s; + try { + s = await stat(p); + } catch { + throw new Error( + `Codebase extra_digest_paths entry not found: ${p} (resolved to ${path.resolve(p)})` + ); + } + return s.isDirectory() + ? await digestDir(p, "") + : await generateHashFromBuffer(await readFile(p)); +} + +/** Bundle inputs that no codebase digest covers, so edits to them would not trigger a re-push. */ +export function uncoveredBundleInputs( + codebase: Codebase, + entry: string, + inputs: string[] +): string[] { + const entryAbs = path.resolve(entry); + const roots = [codebase.relative_path, ...extraDigestPaths(codebase)] + .map((r) => path.resolve(r)); + return inputs.filter((i) => { + // non-file namespaces ("", "(disabled):x", "ns:path") and installed packages + if (/^[<(]|^[a-z-]{2,}:/i.test(i) || i.includes("node_modules")) return false; + const abs = path.resolve(i); + return abs != entryAbs && !roots.some((r) => abs == r || abs.startsWith(r + path.sep)); + }); +} export type SyncCodebase = Codebase & { getDigest: (forceTar?: boolean) => Promise; @@ -23,13 +61,20 @@ export function listSyncCodebases(options: SyncOptions): SyncCodebase[] { forceTar?: boolean ) => { if (_digest == undefined) { - _digest = await digestDir( + // Concurrent callers read `_digest` across these awaits: assign it only once complete. + let digest = await digestDir( codebase.relative_path, JSON.stringify(codebase) ); - if (codebase.format == "esm") { - _digest += ".esm"; + const extra = extraDigestPaths(codebase); + if (extra.length > 0) { + const hashes = await Promise.all(extra.map(digestPath)); + digest = await generateHash(digest + hashes.join("")); } + if (codebase.format == "esm") { + digest += ".esm"; + } + _digest = digest; if (!alreadyPrinted) { alreadyPrinted = true; log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); diff --git a/cli/test/codebase_digest_unit.test.ts b/cli/test/codebase_digest_unit.test.ts new file mode 100644 index 0000000000..f12b928de4 --- /dev/null +++ b/cli/test/codebase_digest_unit.test.ts @@ -0,0 +1,48 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + listSyncCodebases, + uncoveredBundleInputs, +} from "../src/utils/codebase.ts"; + +test("extra_digest_paths edits change the codebase digest", async () => { + const root = await mkdtemp(join(tmpdir(), "wmill-codebase-")); + await mkdir(join(root, "f")); + await mkdir(join(root, "packages/utils"), { recursive: true }); + await writeFile(join(root, "f/main.ts"), "export * from '../packages/utils/a'"); + await writeFile(join(root, "packages/utils/a.ts"), "export const a = 1"); + + const digest = () => + listSyncCodebases({ + codebases: [ + { + relative_path: join(root, "f"), + extra_digest_paths: [join(root, "packages/utils")], + }, + ], + } as any)[0].getDigest(); + + try { + const before = await digest(); + await writeFile(join(root, "packages/utils/a.ts"), "export const a = 2"); + expect(await digest()).not.toBe(before); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("uncoveredBundleInputs reports only files outside the digested roots", () => { + const codebase = { relative_path: "f", extra_digest_paths: ["packages/a"] }; + expect( + uncoveredBundleInputs(codebase, "g/entry.ts", [ + "g/entry.ts", + "f/main.ts", + "packages/a/x.ts", + "packages/ab/y.ts", + "node_modules/pg/index.js", + "", + ]) + ).toEqual(["packages/ab/y.ts"]); +}); diff --git a/cli/wmill.schema.json b/cli/wmill.schema.json index b4b10087a8..022156e5c9 100644 --- a/cli/wmill.schema.json +++ b/cli/wmill.schema.json @@ -147,6 +147,13 @@ "type": "string", "description": "Path to the codebase directory" }, + "extra_digest_paths": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Paths outside relative_path that scripts import, relative to wmill.yaml; hashed recursively" + }, "includes": { "type": "array", "items": { From 2f2882adff8371e33c0fb72cbf0c61298bbcb2ca Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 24 Sep 2026 14:56:24 +0200 Subject: [PATCH 05/91] fix: label chat job links with the end of the job id (#11324) Co-authored-by: Claude Opus 5.5 (1M context) --- .../src/lib/components/copilot/chat/AssistantMessage.svelte | 4 ++-- .../lib/components/copilot/chat/ToolExecutionDisplay.svelte | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte index 603c2c88a4..b6455980a3 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantMessage.svelte @@ -16,7 +16,7 @@ import { Bot, ExternalLink } from 'lucide-svelte' import CopyButton from '$lib/components/common/button/CopyButton.svelte' import { base } from '$lib/base' - import { displayDate } from '$lib/utils' + import { displayDate, truncateRev } from '$lib/utils' interface Props { message: DisplayMessage @@ -191,7 +191,7 @@ class="inline-flex items-center gap-1 hover:text-primary hover:underline" title="Open this run" > - job {jobId?.slice(0, 8)} + job {truncateRev(jobId ?? '', 8)} {/if} diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index a469b3d35c..aa8dc47cec 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -14,6 +14,7 @@ BookOpen } from 'lucide-svelte' import { base } from '$lib/base' + import { truncateRev } from '$lib/utils' import { EXIT_PLAN_MODE_TOOL, isPlanCardTool, @@ -291,7 +292,7 @@ class="shrink-0 inline-flex items-center gap-1 font-main text-2xs text-tertiary hover:text-primary hover:underline" title="Open this run" > - job {message.jobId?.slice(0, 8)} + job {truncateRev(message.jobId ?? '', 8)} {/snippet} From 98d3897b8c926cc6d47a2417b6943a5d5612b3aa Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 24 Sep 2026 15:00:23 +0200 Subject: [PATCH 06/91] fix: hide answer actions until the chat turn ends (#11323) * fix: hide answer actions until the chat turn ends Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep the last turn's answer actions during a manual compaction Co-Authored-By: Claude Opus 5.5 (1M context) * docs: note answer actions stay hidden while a turn waits on the user Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../src/lib/components/copilot/chat/AIChatDisplay.svelte | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 8bb774cfde..2d88d1deb7 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -276,16 +276,22 @@ // run's last answer carries the only link to that run. Keyed on the answer's own job (a step // label repeats when a step runs in a loop), and on answers rather than the next row: a // flow's tool rows have their own job, and a stopped turn can end on a tool row. + // None in a turn still running, paused on the user included (their reply lands as a tool + // result, so a row shown during the pause would vanish again): which answer ends the turn is + // unknown until it does, and one followed by a tool call would leave the row's blank gap above + // it. A manual compaction loads without a turn of its own, so the last turn keeps its row. const showsAnswerActions = $derived.by(() => { const shows: boolean[] = new Array(messages.length).fill(false) const answeredLater = new Set() + let inRunningTurn = chatHost.loading && !chatHost.compacting for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i] if (message.role === 'user' || message.role === 'summary') { answeredLater.clear() + inRunningTurn = false } else if (message.role === 'assistant' && message.content) { const run = message.jobId ?? message.stepName - shows[i] = !answeredLater.has(run) + shows[i] = !inRunningTurn && !answeredLater.has(run) answeredLater.add(run) } } From 852520636160408cd16119184ffe9c7be92682c8 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 24 Sep 2026 15:01:21 +0200 Subject: [PATCH 07/91] fix: pass the schedule's custom tag when using run now (#11322) * fix: pass the schedule's tag when using run now on a schedule Co-Authored-By: Claude Opus 5.5 * fix: keep flow schedules on the flow's own tag for run now Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../components/triggers/scheduled/utils.ts | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/frontend/src/lib/components/triggers/scheduled/utils.ts b/frontend/src/lib/components/triggers/scheduled/utils.ts index a0a1b8daad..064116a194 100644 --- a/frontend/src/lib/components/triggers/scheduled/utils.ts +++ b/frontend/src/lib/components/triggers/scheduled/utils.ts @@ -1,33 +1,33 @@ -import { JobService, ScheduleService } from "$lib/gen" -import { sendUserToast } from "$lib/utils" +import { JobService, ScheduleService } from '$lib/gen' +import { sendUserToast } from '$lib/utils' export async function runScheduleNow( - path: string, - schedulePath: string, - isFlow: boolean, - workspace_id: string + path: string, + schedulePath: string, + isFlow: boolean, + workspace_id: string ): Promise { - try { - const runByPath = isFlow ? JobService.runFlowByPath : JobService.runScriptByPath - const args = ( - await ScheduleService.getSchedule({ - workspace: workspace_id, - path: schedulePath - }) - ).args - const run = await runByPath({ - path, - requestBody: args ?? {}, - workspace: workspace_id - }) + try { + const runByPath = isFlow ? JobService.runFlowByPath : JobService.runScriptByPath + const { args, tag } = await ScheduleService.getSchedule({ + workspace: workspace_id, + path: schedulePath + }) + const run = await runByPath({ + path, + requestBody: args ?? {}, + workspace: workspace_id, + // scheduled flows run on the flow's own tag, like the cron path + tag: (!isFlow && tag) || undefined + }) - sendUserToast(`Schedule ${path} will run now`, false, [ - { - label: 'Go to the run page', - callback: () => window.open('/run/' + run + '?workspace=' + workspace_id, '_blank') - } - ]) - } catch (err) { - sendUserToast(`Cannot run schedule now: ${err.body}`, true) - } -} \ No newline at end of file + sendUserToast(`Schedule ${path} will run now`, false, [ + { + label: 'Go to the run page', + callback: () => window.open('/run/' + run + '?workspace=' + workspace_id, '_blank') + } + ]) + } catch (err) { + sendUserToast(`Cannot run schedule now: ${err.body}`, true) + } +} From 9375c93fd8d55bf22e3f290f44650affdf1868c2 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 24 Sep 2026 09:10:28 -0400 Subject: [PATCH 08/91] fix: trust the system CA store for SMTP TLS (#11328) * fix: trust the system CA store for SMTP TLS Co-Authored-By: Claude Opus 5.5 (1M context) * ci: run the smtp-gated backend tests Co-Authored-By: Claude Opus 5.5 (1M context) * chore: depend on webpki-roots 1 directly Co-Authored-By: Claude Opus 5.5 (1M context) * chore: update ee-repo-ref to 48193da8cb30bc4ae82a50f944caef85d8a20b48 This commit updates the EE repository reference after PR #826 was merged in windmill-ee-private. Previous ee-repo-ref: cd3447143b25d9f3301975feca4b202755f2508f New ee-repo-ref: 48193da8cb30bc4ae82a50f944caef85d8a20b48 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: windmill-internal-app[bot] --- .github/workflows/backend-test.yml | 2 +- backend/Cargo.lock | 5 +++++ backend/Cargo.toml | 1 + backend/ee-repo-ref.txt | 2 +- backend/windmill-common/Cargo.toml | 7 ++++++- 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index fea84e7781..466259f6f8 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -295,4 +295,4 @@ jobs: # never reaches it. Pin the target dir (matching the cache step above) so # its own tests run off this compile rather than a second bundled build. (cd windmill-duckdb-ffi-internal && export CARGO_TARGET_DIR="$PWD/target" && ./build_dev.sh && cargo test --release -p windmill_duckdb_ffi_internal) - DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline --all -- --nocapture --test-threads=10 + DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp --all -- --nocapture --test-threads=10 diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e0bfce842b..57b15bc5a7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15658,11 +15658,14 @@ dependencies = [ "prometheus", "quick_cache", "rand 0.9.5", + "rcgen", "regex", "reqwest 0.13.5", "reqwest-middleware", "reqwest-retry", "rsa", + "rustls 0.23.35", + "rustls-native-certs 0.8.4", "schemars 0.8.22", "semver 1.0.28", "serde", @@ -15683,6 +15686,7 @@ dependencies = [ "tikv-jemalloc-ctl", "tokio", "tokio-postgres", + "tokio-rustls 0.26.5", "tokio-stream", "tokio-util", "tonic 0.13.1", @@ -15693,6 +15697,7 @@ dependencies = [ "url", "urlencoding", "uuid", + "webpki-roots 1.0.9", "windmill-macros", "windmill-parser", "windmill-parser-py", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 74da8a4244..2e81eb2637 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -757,4 +757,5 @@ hyper-http-proxy = { version = "1", default-features = false, features = ["rustl hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "ring", "tls12"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } rustls-native-certs = "0.8" +webpki-roots = "1" rcgen = "0.13" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 211848b65a..9d91911b1d 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ed5a367b4def5280c3a5e1090e7abfa099a01042 +48193da8cb30bc4ae82a50f944caef85d8a20b48 diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 06de7cc4aa..e95fadfe66 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -18,7 +18,7 @@ parquet = [] aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic", "dep:opentelemetry"] -smtp = ["dep:mail-send"] +smtp = ["dep:mail-send", "dep:rustls", "dep:tokio-rustls", "dep:rustls-native-certs", "dep:webpki-roots"] scoped_cache = [] cloud = [] dev_override = [] @@ -85,6 +85,10 @@ aws-sdk-rds = { workspace = true, optional = true } indexmap.workspace = true bytes.workspace = true mail-send = { workspace = true, optional = true } +rustls = { workspace = true, optional = true } +tokio-rustls = { workspace = true, optional = true } +rustls-native-certs = { workspace = true, optional = true } +webpki-roots = { workspace = true, optional = true } futures-core.workspace = true async-stream.workspace = true const_format.workspace = true @@ -136,6 +140,7 @@ equivalent = "1.0.2" [dev-dependencies] # `test-util` is not part of tokio's `full`; it is what lets tests pause the clock. tokio = { workspace = true, features = ["test-util"] } +rcgen.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } From 1b749399521cc5ebaa07fc5a55773113a6a3b903 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:11:26 +0200 Subject: [PATCH 09/91] feat: support aiagent steps in test_run_step (#11321) * fix: support aiagent steps in test_run_step Co-Authored-By: Claude Opus 5.5 (1M context) * fix: offer the agent's tools as a picker in the step run form Co-Authored-By: Claude Opus 5.5 (1M context) * fix: run a blank tool list as no tools when the step form is bypassed Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- frontend/src/lib/components/ModuleTest.svelte | 35 +-- .../lib/components/copilot/chat/flow/core.ts | 12 + .../copilot/chat/global/core.test.ts | 231 ++++++++++++++++++ .../components/copilot/chat/global/core.ts | 90 ++++++- .../src/lib/components/copilot/chat/shared.ts | 58 ++++- .../components/flows/agentFormFields.test.ts | 21 ++ .../lib/components/flows/agentFormFields.ts | 22 ++ .../lib/components/flows/agentToolUtils.ts | 5 + .../flows/content/AiAgentStepInputs.svelte | 6 +- 9 files changed, 438 insertions(+), 42 deletions(-) diff --git a/frontend/src/lib/components/ModuleTest.svelte b/frontend/src/lib/components/ModuleTest.svelte index 491c625d09..e35ac291c9 100644 --- a/frontend/src/lib/components/ModuleTest.svelte +++ b/frontend/src/lib/components/ModuleTest.svelte @@ -4,7 +4,6 @@ type AiAgent, type FlowModule, type InputTransform, - type JavascriptTransform, type Job } from '$lib/gen' import { getScriptByPath } from '$lib/scripts' @@ -20,7 +19,7 @@ type LinkedAgentDraft } from './flows/linkedAgentDrafts' import { AGENT_FLOW_LOCAL_KEYS } from './flows/agentResourceUtils' - import { AGENT_HISTORY_KEYS } from './flows/agentFormFields' + import { agentTestInputTransforms } from './flows/agentFormFields' import { sendUserToast } from '$lib/toast' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' @@ -175,37 +174,17 @@ // `args` spans the whole AI agent schema, so on a linked step it carries every brain key as // undefined; overlaying those would shadow the draft's brain, so an inlined step takes only - // the inputs its form offers. A blank history input is unset, as on the step: an expression - // evaluating to nothing reads as an empty memory id, and the step's transform is stale. - const isBlank = (v: unknown) => v == undefined || v === '' || (Array.isArray(v) && !v.length) - const formKeys = ( - draft ? (AGENT_FLOW_LOCAL_KEYS as readonly string[]) : Object.keys(args) - ).filter( - (key) => !(AGENT_HISTORY_KEYS as readonly string[]).includes(key) || !isBlank(args[key]) - ) - const stepTransforms = Object.fromEntries( - Object.entries((agentVal.input_transforms ?? {}) as Record).filter( - ([key]) => !(AGENT_HISTORY_KEYS as readonly string[]).includes(key) || !isBlank(args[key]) - ) - ) - + // the inputs its form offers. // The test form only covers the schema it was given, and for a standalone agent that may be // the flow-local one (the agent editor shows the brain in its own form, not here). Take the // brain from the module as authored and let the form's own keys win over it, so an edit made // in the form after the test panel mounted is what runs. A linked agent needs none of this: // the server reads its brain from the resource. - const inputTransforms: { [key: string]: JavascriptTransform | InputTransform } = { - ...(agentVal.agent ? {} : stepTransforms), - ...Object.fromEntries( - formKeys.map((key) => [ - key, - { - expr: `flow_input.${key}`, - type: 'javascript' - } - ]) - ) - } + const inputTransforms = agentTestInputTransforms( + agentVal.agent ? {} : ((agentVal.input_transforms ?? {}) as Record), + args, + draft ? AGENT_FLOW_LOCAL_KEYS : Object.keys(args) + ) await jobLoader?.runFlowPreview( args, diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index eef5d396a4..96b2858957 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -464,6 +464,18 @@ export const flowTools: Tool[] = [ const stepId = args.stepId const stepArgs = args.args || {} + // An agent step runs on the step's own configuration with this run's changes on top, which + // the global chat's run form assembles; this tool sends only what the model named. + if (findModuleInFlow(flow.value, stepId)?.value.type === 'aiagent') { + toolCallbacks.setToolStatus(toolId, { + content: `Step "${stepId}" is an AI agent step`, + error: 'AI agent steps cannot be tested from this chat' + }) + throw new Error( + 'AI agent steps cannot be tested from this chat. Test it from the step\'s "Test this step" tab.' + ) + } + return executeFlowStepTestRun({ flowValue: flow.value, stepId, diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 820c593514..e0df5c4a98 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -6086,6 +6086,237 @@ describe('global AI tools', () => { }) }) + // An agent step has no job of its own, so it runs as a flow preview of itself alone, opened on + // the same rows and the same values as the flow editor's own step test. A setting the step + // leaves alone is not offered at all, so a mounted widget cannot invent a value for it: an + // `enabled_tools` the form synthesized empty would advertise none of the agent's tools. + it("test_run_step opens an aiagent step on the step's own configuration", async () => { + const providerValue = { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } + let opened: RunFormDisplay | undefined + await callGlobalTool('write_flow', { + path: 'f/flows/agent-step', + summary: 'Flow with an agent step', + modules: JSON.stringify([ + { + id: 'ask', + value: { + type: 'aiagent', + tools: [ + { + id: 'b', + summary: 'get_code', + value: { + tool_type: 'flowmodule', + type: 'rawscript', + language: 'bun', + content: 'export async function main() { return 1 }', + input_transforms: {} + } + } + ], + input_transforms: { + provider: { type: 'static', value: providerValue }, + user_message: { type: 'static', value: 'authored question' } + } + } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { + path: 'f/flows/agent-step', + stepId: 'ask', + args: { user_message: 'What is the capital of France?' } + }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + opened = form + return form.args + } + } + ) + ) + + // The step configures these two, so the form offers them, filled in with what the step + // holds — and the model's own message wins over the authored one. + expect(opened?.args).toMatchObject({ + provider: providerValue, + user_message: 'What is the capital of France?' + }) + // Those two plus the rows the editor always shows, and nothing else the step is silent + // about: `enabled_tools` above all, whose synthesized empty value would advertise no tools. + expect(Object.keys(opened?.schema?.properties ?? {})).toEqual([ + 'provider', + 'user_message', + 'system_prompt' + ]) + + expect(JobService.runFlowPreview).toHaveBeenCalledWith({ + workspace: WORKSPACE, + requestBody: { + value: { + modules: [ + { + id: 'ask', + value: expect.objectContaining({ + type: 'aiagent', + input_transforms: { + provider: { type: 'javascript', expr: 'flow_input.provider' }, + user_message: { type: 'javascript', expr: 'flow_input.user_message' }, + // Every row of the form runs from it, the blank ones too. + system_prompt: { type: 'javascript', expr: 'flow_input.system_prompt' } + } + }) + } + ] + }, + args: { provider: providerValue, user_message: 'What is the capital of France?' }, + // Filed under the flow, as the editor's step test does: a managed memory keyed on a + // named memory id is found under it. + path: 'f/flows/agent-step' + } + }) + }) + + // What the form submits is what runs, as in the editor's step test: a field cleared over the + // step's expression runs cleared rather than falling back to the expression, whether the + // mounted form sends it as the empty string or the bypass posture sends the form unchanged. + it.each([ + { + answer: 'the mounted form', + // A field the form could not fill in comes back blank, never absent: text as the empty + // string, a list as `[]`. + callbacks: { + requestRunArgs: async (_toolId: string, form: any) => ({ + ...form.args, + user_message: '', + enabled_tools: form.args.enabled_tools ?? [] + }) + } + }, + { + answer: 'the bypass posture', + callbacks: { + shouldAutoAcceptToolConfirmations: () => true, + requestRunArgs: async (_toolId: string, form: any) => form.args + } + } + ])( + 'test_run_step runs an aiagent field left blank by $answer as blank', + async ({ callbacks }) => { + await callGlobalTool('write_flow', { + path: 'f/flows/agent-expr', + summary: 'Agent fed by an earlier step', + modules: JSON.stringify([ + { + id: 'first', + value: { + type: 'rawscript', + language: 'bun', + content: 'export async function main() { return { question: "hi" } }', + input_transforms: {} + } + }, + { + id: 'ask', + value: { + type: 'aiagent', + tools: [], + input_transforms: { + provider: { + type: 'static', + value: { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } + }, + user_message: { type: 'javascript', expr: 'results.first.question' }, + enabled_tools: { type: 'javascript', expr: 'results.first.tools' } + } + } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/agent-expr', stepId: 'ask', args: {} }, + { ...toolCallbacks, ...callbacks } + ) + ) + + const body = vi.mocked(JobService.runFlowPreview).mock.calls.at(-1)?.[0].requestBody as any + expect(body.value.modules[0].value.input_transforms.user_message).toEqual({ + type: 'javascript', + expr: 'flow_input.user_message' + }) + expect(body.args.enabled_tools).toEqual([]) + } + ) + + // The memory picker resets a value whose kind it does not offer to its first option, so a + // step saved with a legacy kind would run with its history switched off. And a tool name typed + // by hand that matches none silently narrows the run, so the roster is offered as in the editor. + it("test_run_step offers an aiagent step the editor's choices", async () => { + let opened: RunFormDisplay | undefined + await callGlobalTool('write_flow', { + path: 'f/flows/agent-legacy-memory', + summary: 'Agent with legacy memory', + modules: JSON.stringify([ + { + id: 'ask', + value: { + type: 'aiagent', + tools: [ + { + id: 'b', + summary: 'get_code', + value: { + tool_type: 'flowmodule', + type: 'rawscript', + language: 'bun', + content: 'export async function main() { return 1 }', + input_transforms: {} + } + } + ], + input_transforms: { + provider: { + type: 'static', + value: { kind: 'openai', resource: '$res:u/admin/openai', model: 'gpt-4o' } + }, + user_message: { type: 'static', value: 'hi' }, + memory: { type: 'static', value: { kind: 'auto', context_length: 5 } }, + enabled_tools: { type: 'static', value: ['get_code'] } + } + } + } + ]) + }) + + await withCompletedTestJob(() => + callGlobalTool( + 'test_run_step', + { path: 'f/flows/agent-legacy-memory', stepId: 'ask', args: {} }, + { + ...toolCallbacks, + requestRunArgs: async (_toolId, form) => { + opened = form + return form.args + } + } + ) + ) + + const kinds = opened?.schema.properties.memory.oneOf.map( + (variant: any) => variant.properties.kind.enum[0] + ) + expect(kinds).toContain('auto') + expect(opened?.schema.properties.enabled_tools.items.enum).toEqual(['get_code']) + }) + // A step is fed by its input transforms, so its arguments are its own and the flow's // schema describes a different set entirely. Opening the form on the flow's would offer // fields this job ignores and drop the ones it takes. diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b646b1283d..5664f2349d 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -43,6 +43,13 @@ import type { Script, ScriptLang } from '$lib/gen/types.gen' +import { loadSchemaFromModule, memoryPropertyFor } from '$lib/components/flows/flowInfers' +import { flowLocalAgentSchema } from '$lib/components/flows/agentResourceUtils' +import { AGENT_FIELDS, initialVisibleAgentFields } from '$lib/components/flows/agentFormFields' +import { withAgentDrafts } from '$lib/components/flows/linkedAgentDrafts' +import { resolveLinkedAgentTools } from '$lib/components/flows/flowState' +import { enabledToolNames, type AgentTool } from '$lib/components/flows/agentToolUtils' +import { evalValue } from '$lib/components/flows/utils.svelte' import { updateRawAppPolicy } from '$lib/components/raw_apps/rawAppPolicy' import { FRAMEWORK_TEMPLATES, @@ -992,7 +999,7 @@ const testRunStepSchema = z.object({ const testRunStepToolDef = createToolDef( testRunStepSchema, 'test_run_step', - "Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists. `args` are the step's OWN inputs, not the flow's: a step is normally fed by its input transforms, so send what that step's code takes, not what the flow takes. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.", + "Execute a test run of one step in a flow by path, preferring draft flow/script content when it exists. `args` are the step's OWN inputs, not the flow's: a step is normally fed by its input transforms, so send what that step's code takes, not what the flow takes. An AI agent step takes the inputs a run supplies rather than code arguments, `user_message` above all; its form opens on the step's own configuration, so send only what this run should change. The user gets an argument form prefilled with `args` and may edit or dismiss it before it runs, so fill in every argument you can infer. For a secret argument prefer `$var:` naming an existing workspace variable; a literal is minted into a short-lived secret before the run, but stays in this call.", { strict: false } ) @@ -5791,6 +5798,68 @@ async function loadSubflowForFlowStep( } } +/** + * The run form for an agent step: the rows the flow editor's own step test shows, opened on the + * values it opens them with, built from that form's helpers so the two cannot drift. + */ +async function agentStepRunForm( + module: FlowModule, + workspace: string, + proposed: Record | null | undefined +): Promise<{ schema: Record; args: Record }> { + const transforms = (module.value as { input_transforms?: Record }).input_transforms + // A linked step's brain belongs to the resource rather than to this flow, so only the inputs + // this flow supplies are its to edit. + const full = (await loadSchemaFromModule(module, workspace)).schema + const schema = (module.value as { agent?: string }).agent ? flowLocalAgentSchema(full) : full + + const visible = initialVisibleAgentFields(transforms, schema.properties) + // What the model asked to set, shown even where the step configures nothing: this form has no + // control for opening a row, so a field left out of it is one the chat cannot reach at all. + for (const key of Object.keys(proposed ?? {})) visible.add(key) + const position = new Map(AGENT_FIELDS.map((field, index) => [field.key, index])) + const keys = Object.keys(schema.properties ?? {}) + // A key the field registry does not know is kept, so a new one is never silently dropped. + .filter((key) => !position.has(key) || visible.has(key)) + .sort((a, b) => (position.get(a) ?? Infinity) - (position.get(b) ?? Infinity)) + + const evaluated: Record = {} + for (const key of keys) { + const value = evalValue(key, module, undefined, false) + if (value !== undefined) evaluated[key] = value + } + const args = { ...evaluated, ...(proposed ?? {}) } + // A mounted form fills a blank list with `[]`, and the bypass posture submits the form unmounted. + // For `enabled_tools` the two mean opposite things: `[]` runs no tools, an absent key runs all. + if (keys.includes('enabled_tools')) args.enabled_tools ??= [] + + const properties = Object.fromEntries(keys.map((key) => [key, schema.properties[key]])) + if (properties.memory) properties.memory = memoryPropertyFor(properties.memory, args.memory) + // Picked from the agent's tools, as in the editor: a name typed by hand that matches none + // silently narrows the run to fewer tools. + const list = properties.enabled_tools + if (list) { + const { agent, tools } = module.value as { agent?: string; tools?: AgentTool[] } + const roster = agent ? await resolveLinkedAgentTools(agent, workspace, true) : (tools ?? []) + properties.enabled_tools = { + ...list, + items: { ...(list.items ?? { type: 'string' }), enum: enabledToolNames(roster) } + } + } + + return { + schema: { + ...schema, + properties, + order: keys, + required: ((schema.required as string[] | undefined) ?? []).filter((key) => + keys.includes(key) + ) + }, + args + } +} + // Leaf of a workspace path (last segment), for human-readable confirmation // prompts. Falls back to the full path, then a generic noun. function pathLeaf(path: unknown, fallback: string): string { @@ -6248,7 +6317,9 @@ async function testRunFlowStepByPath( toolCallbacks, toolId, loadScript: loadScriptForFlowStep, - loadSubflow: loadSubflowForFlowStep + loadSubflow: loadSubflowForFlowStep, + flowPath: args.path, + withAgentDrafts: (value) => withAgentDrafts(value, workspace) }) // The module resolution landed on, not the id that was asked for: the job's entrypoint @@ -6257,15 +6328,20 @@ async function testRunFlowStepByPath( const isPreprocessor = resolved.module.id === SPECIAL_MODULE_IDS.PREPROCESSOR // The step's own inputs, never the flow's: a step is fed by its input transforms, so the // flow's schema names arguments this job would ignore and omits the ones it takes. + const agentForm = + resolved.module.value.type === 'aiagent' + ? await agentStepRunForm(resolved.module, workspace, args.args) + : undefined const schema = - resolved.code != undefined && resolved.lang + agentForm?.schema ?? + (resolved.code != undefined && resolved.lang ? await schemaForTestRun({ content: resolved.code, language: resolved.lang, schema: resolved.schema, entrypoint: isPreprocessor ? 'preprocessor' : undefined }) - : (resolved.schema ?? {}) + : (resolved.schema ?? {})) const stepSummary = resolved.module.summary return runThroughForm( @@ -6283,7 +6359,7 @@ async function testRunFlowStepByPath( // itself be a draft. schemaNoun: 'step', toolName: 'test_run_step', - proposed: args.args, + proposed: agentForm?.args ?? args.args, startMessage: resolved.startMessage, contextName: resolved.runnableKind, noun: 'step', @@ -6292,7 +6368,9 @@ async function testRunFlowStepByPath( autoAcceptable: true, background: args.background, detachAfterMs: waitSecondsToDetachMs(args.wait_seconds), - startJob: resolved.startJob + startJob: agentForm + ? (submitted) => resolved.startJob(submitted, agentForm.schema.order) + : resolved.startJob }, ctx ) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index acb3f95d0d..1e486c7d41 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -39,6 +39,7 @@ import type { CodePieceElement, ContextElement, FlowModuleCodePieceElement } fro import { workspaceStore } from '$lib/stores' import type { ExtendedOpenFlow } from '$lib/components/flows/types' import { findModuleInFlow, findModuleInModules } from '$lib/components/flows/flowTree' +import { agentTestInputTransforms } from '$lib/components/flows/agentFormFields' import type { FunctionParameters } from 'openai/resources/shared.mjs' import { z } from 'zod' import { @@ -2212,6 +2213,13 @@ type FlowStepRunConfig = { toolId: string loadScript?: FlowStepScriptLoader loadSubflow?: FlowStepSubflowLoader + /** The path of the flow the step belongs to. An agent step's preview carries it, as the flow + * editor's own step test does: the run is filed under it, and a managed memory keyed on a + * memory id someone named reads the flow's conversation rather than a stray one. */ + flowPath?: string + /** The flow editor's substitution of a linked agent's unsaved draft, passed in because it + * reaches the draft store, which this module's import list must not. */ + withAgentDrafts?: (value: FlowValue) => Promise } export type FlowStepTestRunConfig = FlowStepRunConfig & { @@ -2236,8 +2244,9 @@ export type ResolvedFlowStepRun = { startMessage: string /** Takes the arguments as submitted. The preprocessor's entrypoint override is added * here rather than by the caller: it is declared by no schema, so anything that - * conforms arguments to one would drop it. */ - startJob: (args: Record) => Promise + * conforms arguments to one would drop it. An agent step also takes the run form's rows: + * each runs from its submitted value, including one the submission leaves out. */ + startJob: (args: Record, formKeys?: readonly string[]) => Promise } function normalizeFlowStepArgs(args: Record | null | undefined): Record { @@ -2281,7 +2290,9 @@ export async function resolveFlowStepRun({ toolCallbacks, toolId, loadScript = loadDeployedScriptForFlowStep, - loadSubflow + loadSubflow, + flowPath, + withAgentDrafts }: FlowStepRunConfig): Promise { const targetModule = findModuleInFlow(flowValue, stepId) ?? undefined @@ -2368,12 +2379,51 @@ export async function resolveFlowStepRun({ } } + if (moduleValue.type === 'aiagent') { + return { + module: targetModule, + // The job started is a flow preview: an agent step has none of its own, and the tool + // calls it makes are that flow's steps. + runnableKind: 'flow', + startMessage: `Starting test run of agent step "${stepId}"...`, + startJob: async (args, formKeys) => { + // The step alone, stripped of the settings around it: a mock would answer for it + // and a suspend would hold the run for an approval nobody is watching for. + const value: FlowValue = { + modules: [ + { + id: targetModule.id, + value: { + ...moduleValue, + input_transforms: agentTestInputTransforms( + moduleValue.input_transforms, + args, + formKeys + ) + } + } + ] + } + return JobService.runFlowPreview({ + workspace, + requestBody: { + // After the inputs are wired, as the flow editor's step test does: a linked agent's + // draft brings its own brain and tools, and the step's run inputs stay on top. + value: withAgentDrafts ? await withAgentDrafts(value) : value, + args, + path: flowPath + } + }) + } + } + } + toolCallbacks.setToolStatus(toolId, { content: `Step type "${moduleValue.type}" not supported for testing`, error: `Cannot test step of type "${moduleValue.type}"` }) throw new Error( - `Cannot test step of type "${moduleValue.type}". Supported types: rawscript, script, flow` + `Cannot test step of type "${moduleValue.type}". Supported types: rawscript, script, flow, aiagent` ) } diff --git a/frontend/src/lib/components/flows/agentFormFields.test.ts b/frontend/src/lib/components/flows/agentFormFields.test.ts index 311b48ade1..48b4c94d92 100644 --- a/frontend/src/lib/components/flows/agentFormFields.test.ts +++ b/frontend/src/lib/components/flows/agentFormFields.test.ts @@ -6,6 +6,7 @@ import { agentMemoryMode, historyInputApplies, agentFieldIsSet, + agentTestInputTransforms, initialVisibleAgentFields } from './agentFormFields' @@ -147,3 +148,23 @@ describe('memoryPropertyFor', () => { ).toBeUndefined() }) }) + +describe('agentTestInputTransforms', () => { + it('unsets a blank history input and feeds every other form key from flow_input', () => { + const authored = { + memory_id: { type: 'static' as const, value: 'thread-1' }, + previous_messages: { type: 'static' as const, value: [{ role: 'user', content: 'hi' }] }, + system_prompt: { type: 'javascript' as const, expr: 'results.a.prompt' } + } + expect( + agentTestInputTransforms(authored, { memory_id: '', previous_messages: [{ role: 'user' }] }, [ + 'memory_id', + 'previous_messages', + 'system_prompt' + ]) + ).toEqual({ + previous_messages: { type: 'javascript', expr: 'flow_input.previous_messages' }, + system_prompt: { type: 'javascript', expr: 'flow_input.system_prompt' } + }) + }) +}) diff --git a/frontend/src/lib/components/flows/agentFormFields.ts b/frontend/src/lib/components/flows/agentFormFields.ts index 3d62497ba3..7f09afdf7e 100644 --- a/frontend/src/lib/components/flows/agentFormFields.ts +++ b/frontend/src/lib/components/flows/agentFormFields.ts @@ -30,6 +30,28 @@ export const AGENT_TOOLS_ROW = 'tools' export const AGENT_HISTORY_KEYS = ['memory_id', 'previous_messages'] as const export type AgentHistoryKey = (typeof AGENT_HISTORY_KEYS)[number] +/** The input transforms a one-off test of an agent step runs with: each form key is fed from + * `flow_input`, over the step's own transforms. A blank history input is unset, as on the step: + * an expression evaluating to nothing reads as an empty memory id, and the step's transform is + * stale. */ +export function agentTestInputTransforms( + stepTransforms: Record | undefined, + args: Record, + formKeys: readonly string[] = Object.keys(args) +): Record { + const isBlank = (v: unknown) => v == undefined || v === '' || (Array.isArray(v) && !v.length) + const kept = (key: string) => + !(AGENT_HISTORY_KEYS as readonly string[]).includes(key) || !isBlank(args[key]) + return { + ...Object.fromEntries(Object.entries(stepTransforms ?? {}).filter(([key]) => kept(key))), + ...Object.fromEntries( + formKeys + .filter(kept) + .map((key) => [key, { type: 'javascript' as const, expr: `flow_input.${key}` }]) + ) + } +} + /** What turning managed memory on writes. */ export const DEFAULT_AGENT_MEMORY: MemoryConfig = { kind: 'window', context_length: 10 } diff --git a/frontend/src/lib/components/flows/agentToolUtils.ts b/frontend/src/lib/components/flows/agentToolUtils.ts index e9838f60e3..87c15bc66a 100644 --- a/frontend/src/lib/components/flows/agentToolUtils.ts +++ b/frontend/src/lib/components/flows/agentToolUtils.ts @@ -126,6 +126,11 @@ export function toolEnabledName(tool: AgentTool): string | undefined { return toolDisplayName(tool) } +/** The names `enabled_tools` can pick from: every tool in the roster that has one. */ +export function enabledToolNames(tools: AgentTool[]): string[] { + return tools.map((tool) => toolEnabledName(tool)).filter((name): name is string => !!name) +} + /** * Create an AI Agent tool (nested agent) */ diff --git a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte index 33a15922f5..bb0f260a21 100644 --- a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte +++ b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte @@ -48,7 +48,7 @@ import { AlertTriangle, Plus, X } from 'lucide-svelte' import type { PickableProperties } from '../previousResults' import type { FlowCopilotContext } from '$lib/components/copilot/flow' - import { toolEnabledName, type AgentTool } from '../agentToolUtils' + import { enabledToolNames, type AgentTool } from '../agentToolUtils' import { AGENT_FIELDS, AGENT_FIELD_BY_KEY, @@ -216,9 +216,7 @@ // By what each tool is named, not the summary alone: an MCP entry is added without one and is // named by its resource path, so keying on `summary` would leave a whole server with no name // to pick. `narrow_roster` matches that path for the same reason. - const names = tools - .map((tool) => toolEnabledName(tool)) - .filter((name): name is string => !!name) + const names = enabledToolNames(tools) const properties = schemaProperties untrack(() => { const list = properties['enabled_tools'] From d02ff9ac24d6d720c478e8424e73eebae9ec98fe Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 24 Sep 2026 15:12:09 +0200 Subject: [PATCH 10/91] fix: centre the toggle knob inside its track (#11314) * fix: centre the toggle knob inside its track The knob was positioned against the toggle's wrapper rather than the track, so its 2px inset resolved to a 1px gap inside the track's 1px border. On a 1x screen a toggle that lands on a fractional x blurs its edges across a whole pixel, which swallows that 1px gap and makes the knob look like it overflows the track. Position the knob inside the track's border with a 2px gap on every side, size the toggle in whole px so the 18px root font of large screens cannot make the track fractional, and give each size an explicit checked translate now that the knob is no longer exactly one translate-x-full wide. Co-Authored-By: Claude Opus 5 (1M context) * refactor: hoist the toggle knob's shared inset out of the size branches Co-Authored-By: Claude Opus 5 (1M context) * chore(frontend): add a Toggles tab to the kitchen sink Covers the four sizes, the three colors, the label and EE-badge variants, and a grid of quarter-pixel offsets: the knob's gap against the track border only misreads once the track lands between device pixels. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/components/Toggle.svelte | 16 ++-- frontend/src/routes/kitchen_sink/+page.svelte | 83 +++++++++++++++++++ 2 files changed, 94 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/Toggle.svelte b/frontend/src/lib/components/Toggle.svelte index 16e1086a94..200b449ee7 100644 --- a/frontend/src/lib/components/Toggle.svelte +++ b/frontend/src/lib/components/Toggle.svelte @@ -93,6 +93,8 @@ +
    +
    diff --git a/frontend/src/routes/kitchen_sink/+page.svelte b/frontend/src/routes/kitchen_sink/+page.svelte index b3ae4ef196..e21f9da550 100644 --- a/frontend/src/routes/kitchen_sink/+page.svelte +++ b/frontend/src/routes/kitchen_sink/+page.svelte @@ -4,6 +4,7 @@ import TabContent from '$lib/components/common/tabs/TabContent.svelte' import Tabs from '$lib/components/common/tabs/Tabs.svelte' import DarkModeToggle from '$lib/components/sidebar/DarkModeToggle.svelte' + import Toggle from '$lib/components/Toggle.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import AssistantMessage from '$lib/components/copilot/chat/AssistantMessage.svelte' import type { DisplayMessage } from '$lib/components/copilot/chat/shared' @@ -133,12 +134,23 @@ That's the full round-trip.` role: 'assistant', content: chatSampleContent } + + const toggleSizes = ['2xs', 'xs', 'sm', 'md'] as const + const toggleColors = ['blue', 'red', 'nord'] as const + // The knob's gap against the track border is thin enough that it only misreads once the + // track lands between device pixels, which needs a 1x screen. These offsets put one row of + // each size on every quarter-pixel so the worst case is on the page rather than left to + // whichever label widths a real view happens to have. + const subpixelOffsets = [0, 0.25, 0.5, 0.75] + let toggleChecked = $state(true) + let toggleDisabled = $state(false) + @@ -195,6 +207,77 @@ That's the full round-trip.`
    + +
    + + +
    + +
    +
    Sizes
    +
    + {#each toggleSizes as size} + + {/each} +
    +
    + +
    +
    Colors
    +
    + {#each toggleColors as color} + + {/each} +
    +
    + +
    +
    Labels
    +
    + + + + +
    +
    + +
    +
    Quarter-pixel offsets
    +
    + Zoom a 1x screen in on these: the knob must keep a visible ring of track on every side in + both states. It reads as overflowing the track once that ring drops below a device pixel. +
    +
    + {#each toggleSizes as size} +
    + {#each subpixelOffsets as offset} +
    + +
    + {/each} +
    + {/each} +
    +
    +
    The three markdownProse presets, same source. Each is sized for its own From ebb3048ca0b6cb9f3c9130ac0871581f3c2ed307 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:14:59 +0200 Subject: [PATCH 11/91] feat: mount session list pages in process instead of iframes (#11289) * feat: mount session list pages in process instead of iframes * fix: seed filter defaults from the query, not the cached search params * fix: type trigger list rows from the generated trigger types * fix: judge hosted lists by the operating workspace's rules and keep role-gated filters * fix: keep the user folders filter key for every role, hidden where it does not apply * fix: drop the user folders filter for users it is not offered to * fix: wait for a known user before dropping the user folders filter * style: tint trigger rows for every kind and align the schedules footer wording * fix: stamp edited_at when a schedule is updated --------- Co-authored-by: Ruben Fiszel --- ...8825cdd7784d0d611cd0069b15d11039ac56.json} | 4 +- backend/windmill-api-schedule/src/lib.rs | 1 + .../ContextualVariableEditor.svelte | 6 +- .../src/lib/components/DeployWorkspace.svelte | 47 +- .../src/lib/components/FilterSearchbar.svelte | 4 +- .../lib/components/NoDirectDeployAlert.svelte | 52 +- .../components/ResourceEditorDrawer.svelte | 2 - frontend/src/lib/components/ShareModal.svelte | 39 +- .../src/lib/components/SupabaseConnect.svelte | 9 +- .../src/lib/components/VariableEditor.svelte | 2 - .../lib/components/home/ListFilters.svelte | 62 +- frontend/src/lib/components/hostedPage.ts | 31 + .../src/lib/components/jobs/JobPreview.svelte | 6 +- .../components/resources/ResourcesList.svelte | 1623 +++++++++++++++++ .../components/resources/resourcesFilter.ts | 19 +- .../components/schedules/SchedulesList.svelte | 718 ++++++++ .../components/schedules/schedulesFilter.ts | 19 +- .../components/sessions/PageListView.svelte | 131 ++ .../components/sessions/PreviewTabHost.svelte | 26 + .../components/sessions/pageDrawerSession.ts | 29 - .../lib/components/sessions/previewPaths.ts | 5 + .../components/sessions/previewRouter.test.ts | 11 + .../lib/components/sessions/previewRouter.ts | 21 +- .../sessions/sessionRuntime.svelte.ts | 3 +- .../components/triggers/TriggerList.svelte | 983 ++++++++++ .../amqp/AmqpTriggerEditorInner.svelte | 2 - .../azure/AzureTriggerEditorInner.svelte | 2 - .../email/EmailTriggerEditorInner.svelte | 2 - .../triggers/gcp/GcpTriggerEditorInner.svelte | 2 - .../triggers/http/RouteEditorInner.svelte | 2 - .../triggers/http/RoutesGenerator.svelte | 12 +- .../kafka/KafkaTriggerEditorInner.svelte | 2 - .../mqtt/MqttTriggerEditorInner.svelte | 2 - .../nats/NatsTriggerEditorInner.svelte | 2 - .../PostgresTriggerEditorInner.svelte | 2 - .../schedules/ScheduleEditorInner.svelte | 2 - .../triggers/sqs/SqsTriggerEditorInner.svelte | 2 - .../components/triggers/triggerListConfig.ts | 506 +++++ .../WebsocketTriggerEditorInner.svelte | 2 - .../components/variables/VariablesList.svelte | 759 ++++++++ .../components/variables/variablesFilter.ts | 19 +- frontend/src/lib/svelte5UtilsKit.svelte.ts | 21 +- .../src/routes/(root)/(logged)/+layout.svelte | 21 +- .../(logged)/amqp_triggers/+page.svelte | 569 +----- .../(logged)/azure_triggers/+page.svelte | 667 +------ .../(logged)/email_triggers/+page.svelte | 552 +----- .../(root)/(logged)/gcp_triggers/+page.svelte | 653 +------ .../(logged)/kafka_triggers/+page.svelte | 594 +----- .../(logged)/mqtt_triggers/+page.svelte | 569 +----- .../(logged)/nats_triggers/+page.svelte | 591 +----- .../(logged)/postgres_triggers/+page.svelte | 673 +------ .../(root)/(logged)/resources/+page.svelte | 1587 +--------------- .../(root)/(logged)/routes/+page.svelte | 591 +----- .../(root)/(logged)/schedules/+page.svelte | 682 +------ .../(root)/(logged)/sessions/+page.svelte | 13 +- .../(root)/(logged)/sqs_triggers/+page.svelte | 566 +----- .../(root)/(logged)/variables/+page.svelte | 721 +------- .../(logged)/websocket_triggers/+page.svelte | 578 +----- 58 files changed, 5057 insertions(+), 9764 deletions(-) rename backend/.sqlx/{query-bd2609f999054c5a2a85f2aafe4005bc22731c27f0bf27b3b06ec426a05eda2e.json => query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json} (82%) create mode 100644 frontend/src/lib/components/hostedPage.ts create mode 100644 frontend/src/lib/components/resources/ResourcesList.svelte create mode 100644 frontend/src/lib/components/schedules/SchedulesList.svelte create mode 100644 frontend/src/lib/components/sessions/PageListView.svelte create mode 100644 frontend/src/lib/components/triggers/TriggerList.svelte create mode 100644 frontend/src/lib/components/triggers/triggerListConfig.ts create mode 100644 frontend/src/lib/components/variables/VariablesList.svelte diff --git a/backend/.sqlx/query-bd2609f999054c5a2a85f2aafe4005bc22731c27f0bf27b3b06ec426a05eda2e.json b/backend/.sqlx/query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json similarity index 82% rename from backend/.sqlx/query-bd2609f999054c5a2a85f2aafe4005bc22731c27f0bf27b3b06ec426a05eda2e.json rename to backend/.sqlx/query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json index 578edc6044..5c0e45b0d4 100644 --- a/backend/.sqlx/query-bd2609f999054c5a2a85f2aafe4005bc22731c27f0bf27b3b06ec426a05eda2e.json +++ b/backend/.sqlx/query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE schedule SET\n schedule = $1,\n timezone = $2,\n args = $3,\n on_failure = $4,\n on_failure_times = $5,\n on_failure_exact = $6,\n on_failure_extra_args = $7,\n on_recovery = $8,\n on_recovery_times = $9,\n on_recovery_extra_args = $10,\n on_success = $11,\n on_success_extra_args = $12,\n ws_error_handler_muted = $13,\n retry = $14,\n summary = $15,\n no_flow_overlap = $16,\n tag = $17,\n paused_until = $18,\n path = $19,\n workspace_id = $20,\n cron_version = COALESCE($21, cron_version),\n description = $22,\n dynamic_skip = $23,\n email = $24,\n edited_by = $25,\n permissioned_as = $26,\n labels = COALESCE($27, labels)\n WHERE path = $19 AND workspace_id = $20\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels\n ", + "query": "\n UPDATE schedule SET\n schedule = $1,\n timezone = $2,\n args = $3,\n on_failure = $4,\n on_failure_times = $5,\n on_failure_exact = $6,\n on_failure_extra_args = $7,\n on_recovery = $8,\n on_recovery_times = $9,\n on_recovery_extra_args = $10,\n on_success = $11,\n on_success_extra_args = $12,\n ws_error_handler_muted = $13,\n retry = $14,\n summary = $15,\n no_flow_overlap = $16,\n tag = $17,\n paused_until = $18,\n path = $19,\n workspace_id = $20,\n cron_version = COALESCE($21, cron_version),\n description = $22,\n dynamic_skip = $23,\n email = $24,\n edited_by = $25,\n edited_at = now(),\n permissioned_as = $26,\n labels = COALESCE($27, labels)\n WHERE path = $19 AND workspace_id = $20\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels\n ", "describe": { "columns": [ { @@ -236,5 +236,5 @@ true ] }, - "hash": "bd2609f999054c5a2a85f2aafe4005bc22731c27f0bf27b3b06ec426a05eda2e" + "hash": "4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56" } diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index dfd4a6e3ba..78cd6eb378 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -619,6 +619,7 @@ async fn edit_schedule( dynamic_skip = $23, email = $24, edited_by = $25, + edited_at = now(), permissioned_as = $26, labels = COALESCE($27, labels) WHERE path = $19 AND workspace_id = $20 diff --git a/frontend/src/lib/components/ContextualVariableEditor.svelte b/frontend/src/lib/components/ContextualVariableEditor.svelte index 43277787ba..9adde76e74 100644 --- a/frontend/src/lib/components/ContextualVariableEditor.svelte +++ b/frontend/src/lib/components/ContextualVariableEditor.svelte @@ -1,7 +1,6 @@ -{#if !$userStore?.operator && activeDeployRulesets.length > 0} +{#if !operatingUser.current?.operator && activeDeployRulesets.length > 0}
    { - if (handOffPageDrawer(RESOURCES_PATH, p)) return // A `close({ keepAnchor })` on an already-closed drawer emits no close event, so the flag // would still be standing when the next drawer session ends and would swallow that one's // anchor clear. Every session starts having to clear its own. diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index 471ed74099..da6cc9a674 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -10,7 +10,7 @@ } from '$lib/gen' import type { Folder } from '$lib/gen/types.gen' import { createEventDispatcher } from 'svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { Alert, Button, Drawer } from './common' import DrawerContent from './common/drawer/DrawerContent.svelte' import { sendUserToast } from '$lib/toast' @@ -22,11 +22,22 @@ import Toggle from './Toggle.svelte' import { Trash } from 'lucide-svelte' import { DEMO_RESTRICTION_HINT, isDemoWorkspaceRestricted } from '$lib/cloud' + import { + useOperatingWorkspace, + useOperatingUser + } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() const dispatch = createEventDispatcher() let restricted = $derived( - isDemoWorkspaceRestricted($workspaceStore, $userStore?.is_admin, $userStore?.is_super_admin) + isDemoWorkspaceRestricted( + $operatingWorkspace, + operatingUser.current?.is_admin, + operatingUser.current?.is_super_admin + ) ) type Kind = @@ -81,7 +92,7 @@ defaultPermsLabel = `Folder f/${folderName} permissions` try { const folder: Folder = await FolderService.getFolder({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, name: folderName }) if (path !== currentPath) return @@ -134,7 +145,7 @@ const currentPath = path try { const resource = await ResourceService.getResource({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path: currentPath }) if (path !== currentPath) return @@ -164,27 +175,27 @@ } async function loadOwner() { - own = isOwner(path, $userStore!, $workspaceStore!) + own = isOwner(path, operatingUser.current!, $operatingWorkspace!) } async function loadAcls() { acls = Object.entries( - await GranularAclService.getGranularAcls({ workspace: $workspaceStore!, path, kind }) + await GranularAclService.getGranularAcls({ workspace: $operatingWorkspace!, path, kind }) ) } async function loadGroups(): Promise { - groups = await GroupService.listGroupNames({ workspace: $workspaceStore! }) + groups = await GroupService.listGroupNames({ workspace: $operatingWorkspace! }) } async function loadUsernames(): Promise { - usernames = await UserService.listUsernames({ workspace: $workspaceStore! }) + usernames = await UserService.listUsernames({ workspace: $operatingWorkspace! }) } async function deleteAcl(owner: string) { try { await GranularAclService.removeGranularAcls({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path, kind, requestBody: { owner } @@ -193,7 +204,7 @@ for (const varPath of linkedVarPaths) { try { await GranularAclService.removeGranularAcls({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path: varPath, kind: 'variable', requestBody: { owner } @@ -212,7 +223,7 @@ async function addAcl(owner: string, write: boolean) { await GranularAclService.addGranularAcls({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path, kind, requestBody: { owner, write } @@ -221,7 +232,7 @@ for (const varPath of linkedVarPaths) { try { await GranularAclService.addGranularAcls({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, path: varPath, kind: 'variable', requestBody: { owner, write } @@ -251,9 +262,7 @@
    {/if}
    - Extra members ({acls?.length ?? 0}) + Extra members ({acls?.length ?? 0}) {#if linkedVarPaths.length > 0}
    void = (url: URL) => { - if (syncQuery) { - window.history.pushState(history.state, '', `?${url?.searchParams.toString()}`) - } - } - - const eventListener = (e: PopStateEvent) => { - if (syncQuery) { - loadFilterFromUrl() - } - } - - onMount(() => { - window.addEventListener('popstate', eventListener) - }) - - onDestroy(() => { - window.removeEventListener('popstate', (e) => eventListener(e)) - }) - - loadFilterFromUrl() + const hosted = useHostedPage() function loadFilterFromUrl() { - let queryValue = new URL(window.location.href).searchParams.get(queryName) ?? undefined - selectedFilter = queryValue + const search = hosted ? hosted.search : window.location.search + selectedFilter = new URLSearchParams(search).get(queryName) ?? undefined } - export async function setQuery(url: URL, key: string, value: string | undefined): Promise { - if (value != undefined) { - url.searchParams.set(key, value) - } else { - url.searchParams.delete(key) - } - queryChange(url) + function writeQuery(value: string | undefined) { + if (!syncQuery) return + const params = new URLSearchParams(hosted ? hosted.search : window.location.search) + if (value != undefined) params.set(queryName, value) + else params.delete(queryName) + if (hosted) hosted.setSearch(params.size ? `?${params}` : '') + else window.history.pushState(history.state, '', `?${params}`) } + $effect(() => { + if (!syncQuery) return + if (hosted) { + hosted.search + untrack(loadFilterFromUrl) + return + } + window.addEventListener('popstate', loadFilterFromUrl) + return () => window.removeEventListener('popstate', loadFilterFromUrl) + }) + + untrack(loadFilterFromUrl) + let filtersAndSelected = $derived( selectedFilter ? filters.includes(selectedFilter) @@ -109,11 +103,7 @@ class="inline-flex items-center gap-1 align-middle" onclick={() => { selectedFilter = selectedFilter == filter ? undefined : filter - if (selectedFilter) { - setQuery(new URL(window.location.href), queryName, selectedFilter) - } else { - setQuery(new URL(window.location.href), queryName, undefined) - } + writeQuery(selectedFilter) }} {color} clickable diff --git a/frontend/src/lib/components/hostedPage.ts b/frontend/src/lib/components/hostedPage.ts new file mode 100644 index 0000000000..424c10c450 --- /dev/null +++ b/frontend/src/lib/components/hostedPage.ts @@ -0,0 +1,31 @@ +import { getContext, setContext } from 'svelte' + +// A list page mounted somewhere other than its own route — an AI session's preview tab. The +// document URL there belongs to the host, so the page's view (the filters in its query) has to +// live wherever the host keeps it, and its rows and links open where the host decides instead of +// in a drawer or a top-level navigation. Components under such a host read this instead of +// `window.location`; outside one there is none, and they keep to the document. + +export type HostedPage = { + /** The page's query, `''` or `?…`. Reactive: the host may re-point it. */ + readonly search: string + /** Replace the page's query. */ + setSearch(search: string): void + /** Open one of the page's rows, by path. */ + openItem(path: string): void + /** Follow an in-app link (base included) the page would otherwise navigate to. */ + openLink(href: string): void +} + +const KEY = Symbol('hostedPage') + +/** Reads context: call during component initialisation. */ +export function setHostedPage(host: HostedPage): void { + setContext(KEY, host) +} + +/** The host of this page, or undefined on the page's own route. Reads context: call during + * component initialisation. */ +export function useHostedPage(): HostedPage | undefined { + return getContext(KEY) +} diff --git a/frontend/src/lib/components/jobs/JobPreview.svelte b/frontend/src/lib/components/jobs/JobPreview.svelte index 972cff6799..7f95ef8f63 100644 --- a/frontend/src/lib/components/jobs/JobPreview.svelte +++ b/frontend/src/lib/components/jobs/JobPreview.svelte @@ -15,8 +15,10 @@ import { Badge } from '../common' import { forLater } from '$lib/forLater' import DurationMs from '../DurationMs.svelte' - import { workspaceStore } from '$lib/stores' import { twMerge } from 'tailwind-merge' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + + const operatingWorkspace = useOperatingWorkspace() const POPUP_HEIGHT = 320 as const @@ -115,7 +117,7 @@ Arguments
    diff --git a/frontend/src/lib/components/resources/ResourcesList.svelte b/frontend/src/lib/components/resources/ResourcesList.svelte new file mode 100644 index 0000000000..180474276a --- /dev/null +++ b/frontend/src/lib/components/resources/ResourcesList.svelte @@ -0,0 +1,1623 @@ + + + { + deleteConfirmedCallback = undefined + }} + on:confirmed={() => { + if (deleteConfirmedCallback) { + deleteConfirmedCallback() + } + deleteConfirmedCallback = undefined + }} +> +
    + Are you sure you want to remove {deletePath}? + {#if deleteIsLinked} + + This resource is linked with a variable of the same path. That variable is deleted with it, + unless another resource still references it. + + {/if} + +
    + You can press + SHIFT + while removing a resource to bypass confirmation. +
    +     +
    +     + + + + + inferrer?.toggleDrawer?.()}> + + {#snippet actions()} + + {/snippet} + + + + + + {#snippet titleExtra()} + + {/snippet} +
    + {#if resourceTypeViewerObj.description} + + {/if} + {#if resourceTypeViewerObj.isFileset} + + This resource type represents a collection of files. Each file is identified by its + relative path and contains text content. In the CLI, filesets are stored as directories. + + {:else if resourceTypeViewerObj.formatExtension} + + This resource type represents a plain text file with a .{resourceTypeViewerObj.formatExtension} + extension. (e.g. my_file.{resourceTypeViewerObj.formatExtension}) + + {:else} + + {/if} +
    +     +     + + + + {#snippet actions()} + + {/snippet} +
    + + +
    + {#if editResourceType.isFileset} + + This resource type represents a collection of files. The schema cannot be edited. + + {:else if editResourceType.formatExtension} + + This resource type represents a plain text file with a .{editResourceType.formatExtension} + extension. (e.g. my_file.{editResourceType.formatExtension}). The schema only + contains a `content` field and thus cannot be edited. + + {:else} +
    Schema
    +
    + +
    + {/if} +
    +
    +     +     + + + + {#snippet actions()} + + {/snippet} +
    + + +
    +
    +
    Schema
    +
    + +
    +
    + +
    + +
    +
    +
    +     +     + +{#if operatingUser.current?.operator && $operatingWorkspace && !$userWorkspaces.find((_) => _.id === $operatingWorkspace)?.operator_settings?.resources} + +{:else} + + + {#if showCreateButtons} +
    + + +
    + {/if} +     + (showCreateButtons = v)} /> +
    + { + if (e.detail == 'cache') { + loading.resources = true + loadCache() + } else if (e.detail == 'states') { + loading.resources = true + loadState() + } else if (e.detail == 'theme') { + loading.resources = true + loadTheme() + } + }} + > + + + {#snippet extra()} + + Every resource has a Resource Type attached to it which contains its schema and make + it easy in scripts and flows to accept only resources of a specific resource type. + + {/snippet} + + + {#snippet extra()} + + States are actually resources (but excluded from the Workspace tab for clarity). + States are used by scripts to keep data persistent between runs of the same script by + the same trigger (schedule or user) + + {/snippet} + + + {#snippet extra()} + + Cached results are actually resources (but excluded from the Workspace tab for + clarity). Cache are used by flows's step to cache result to avoid recomputing + unnecessarily + + {/snippet} + + + {#snippet extra()} + + Themes are actually resources (but excluded from the Workspace tab for clarity). They + are CSS for the legacy low-code app editor only, and are added from the CSS panel of + an app rather than from this page. + + {/snippet} + + +
    +
    +
    + + {#if showTable} +
    + {#if loading.resources} + + {#each new Array(6) as _} + + {/each} + {:else if filteredItems?.length == 0} + {#if activeFilters} + + {:else} + appConnect?.open?.(), + aiId: 'resources-empty-add-resource', + aiDescription: 'Add resource' + } + : undefined} + /> + {/if} + {:else} + + + + + Path + Resource type + Description + Status + Actions + + + + {#if filteredItems} + {#each filteredItems as { path, description, resource_type, extra_perms, canWrite, is_oauth, is_linked, account, refresh_error, is_expired, marked, is_refreshed, labels, inherited_labels, ws_specific, draft_only, is_draft }} + {@const hasDraft = + getLocalDraftHint($operatingWorkspace, 'resource', path) ?? is_draft} + + + + + +
    + { + handledHash = `#/resource/${path}` + openResourceEditor(path, resource_type) + }} + >{#if marked}{@html marked}{:else}{path}{/if}{hasDraft ? '*' : ''} + + {#if labels?.length} +
    + {#each labels as label} + { + const arr = (filters.val.label ?? '').split(',').filter(Boolean) + const idx = arr.indexOf(label) + if (idx >= 0) arr.splice(idx, 1) + else arr.push(label) + const newFilters = { ...filters.val } + if (arr.length) newFilters.label = arr.join(',') + else delete newFilters.label + filters.val = newFilters + }}>{label} + {/each} +
    + {/if} + +
    +     + + { + const linkedRt = resourceTypes?.find((rt) => rt.name === resource_type) + if (linkedRt) { + resourceTypeViewerObj = { + rt: linkedRt.name, + //@ts-ignore + schema: linkedRt.schema, + description: linkedRt.description ?? '', + formatExtension: linkedRt.format_extension, + isFileset: linkedRt.is_fileset ?? false + } + resourceTypeViewer?.openDrawer?.() + } else { + sendUserToast( + `Resource type ${resource_type} not found in workspace.`, + true + ) + } + }} + > + + + + +
    + + {removeMarkdown(truncate(description ?? '', 200))} + +
    +     + +
    +
    + {#if is_linked} + + + {#snippet text()} +
    + This resource is linked with a variable of the same path. They + are deleted and renamed together. +
    + {/snippet} +     + {/if} +
    +
    + {#if is_refreshed} + + + {#snippet text()} +
    + The OAuth token will be kept up-to-date in the background by + Windmill using its refresh token +
    + {/snippet} +     + {/if} +
    + + {#if is_oauth} +
    + {#if refresh_error} + + + {#snippet text()} +
    + Latest exchange of the refresh token did not succeed. Error: {refresh_error} +
    + {/snippet} +     + {:else if is_expired} + + + + {#snippet text()} +
    + The access_token is expired, it will get renewed the next time + this variable is fetched or you can request is to be refreshed + in the dropdown on the right. +
    + {/snippet} +     + {:else} + + + {#snippet text()} +
    + The resource was connected through OAuth and the token is not + expired. +
    + {/snippet} +     + {/if} +
    + {/if} +
    +     + +
    + {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !operatingUser.current?.operator} + + {/if} + { + shareModal?.openDrawer?.(path, 'resource') + } + }, + { + displayName: 'Edit', + icon: Pen, + disabled: !canWrite || !showCreateButtons, + action: () => { + openResourceEditor(path, resource_type) + } + }, + // The agent form covers an agent's configuration, not everything a + // resource carries: the workspace-specific toggle in particular is + // only in the generic editor. JSON rather than that editor's form, + // which would render the configuration field by field and write a + // default into every one the agent leaves unset. Both write the same + // draft row, so this is a second view of the same edits. + ...(resource_type === 'ai_agent' + ? [ + { + displayName: 'Edit as JSON', + icon: Braces, + disabled: !canWrite || !showCreateButtons, + action: () => { + // The drawer anchors itself in the hash, which the deep-link + // effect would then read and route back to the agent editor. + // Claim it first, as the row's own link does. + handledHash = `#/resource/${path}` + // A host's item tab opens an agent on JSON already. + if (hosted) hosted.openItem(path) + else resourceEditor?.initEdit?.(path, { json: true }) + } + } + ] + : []), + ...(!ws_specific && isDeployable('resource', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'resource') + } + } + ] + : []), + { + displayName: 'Delete', + disabled: !canWrite || !showCreateButtons, + icon: Trash, + type: 'delete', + action: (event) => { + // TODO + // @ts-ignore + if (event?.shiftKey) { + deleteResource(path, account) + } else { + deleteIsLinked = is_linked ?? false + deletePath = path + deleteConfirmedCallback = () => { + deleteResource(path, account) + } + } + } + }, + ...(account != undefined + ? [ + { + displayName: 'Refresh token', + icon: RotateCw, + action: async () => { + await OauthService.refreshToken({ + workspace: $operatingWorkspace ?? '', + id: account ?? 0, + requestBody: { + path + } + }) + sendUserToast('Token refreshed') + loadResources() + } + } + ] + : []) + ]} + /> +
    +     + {/each} + {/if} + +     + {/if} +
    + {:else if tab == 'types'} + {#if loading.types} + + {#each new Array(6) as _} + + {/each} + {:else if filteredResourceTypes?.length == 0} +
    + +
    + {:else} +
    + + + + Name + Description + Actions + + + + {#if filteredResourceTypes} + {#each filteredResourceTypes as { name, description, schema, canWrite, format_extension, is_fileset }} + + + { + resourceTypeViewerObj = { + rt: name, + //@ts-ignore + schema: schema, + description: description ?? '', + formatExtension: format_extension, + isFileset: is_fileset ?? false + } + + resourceTypeViewer?.openDrawer?.() + }} + > + + + + + +
    + + {removeMarkdown(truncate(description ?? '', 200))} + +
    +     + + {#if !canWrite} + +
    + + Shared globally + + This resource type is from the 'admins' workspace shared with all + workspaces + + +
    + {:else if operatingUser.current?.is_admin || operatingUser.current?.is_super_admin} +
    + + +
    + {:else} + + Non Editable + + Since resource types are shared with the whole workspace, only admins + can edit/delete them + + + {/if} +     +     + {/each} + {/if} + +     +
    + {/if} + {/if} +     +     +{/if} + + + + + + + { + loadResources() + }} +/> + + + +{#if !hosted && agentEditorTarget()} + {#await import('$lib/components/flows/content/AgentEditorModal.svelte') then { default: AgentEditorModal }} + t.host === undefined} + onRenamed={(from, to) => { + void loadResources() + // Only while the dialog still shows the agent: closed mid-request, it already cleared the + // anchor, and writing it back would reopen the editor on refresh. + if (agentEditorTarget()?.path !== from) return + // Claimed first, as a row click does, so the deep-link effect does not reopen it. + handledHash = `#/resource/${to}` + setPageDrawerAnchor(RESOURCES_PATH, to) + }} + /> + {/await} +{/if} diff --git a/frontend/src/lib/components/resources/resourcesFilter.ts b/frontend/src/lib/components/resources/resourcesFilter.ts index 0526ff87b3..97d1b61c7e 100644 --- a/frontend/src/lib/components/resources/resourcesFilter.ts +++ b/frontend/src/lib/components/resources/resourcesFilter.ts @@ -76,15 +76,14 @@ export function buildResourcesFilterSchema({ icon: Tag, description: 'Filter by label (comma-separated for multiple)' }, - ...(showUserFoldersFilter - ? { - user_folders_only: { - type: 'boolean' as const, - label: userFoldersLabel || 'User folders only', - icon: Users, - description: 'Show only resources in user folders' - } - } - : {}) + // The key stays for every role, hidden where the filter does not apply: the list's URL sync + // reads the key set once, before the acting user that gates it may have loaded. + user_folders_only: { + type: 'boolean' as const, + label: userFoldersLabel || 'User folders only', + icon: Users, + description: 'Show only resources in user folders', + hidden: !showUserFoldersFilter + } } satisfies FilterSchemaRec } diff --git a/frontend/src/lib/components/schedules/SchedulesList.svelte b/frontend/src/lib/components/schedules/SchedulesList.svelte new file mode 100644 index 0000000000..80236a300f --- /dev/null +++ b/frontend/src/lib/components/schedules/SchedulesList.svelte @@ -0,0 +1,718 @@ + + + + + +{#if operatingUser.current?.operator && $operatingWorkspace && !$userWorkspaces.find((_) => _.id === $operatingWorkspace)?.operator_settings?.schedules} + +{:else} + + + + +
    +
    + + {#snippet children({ item })} + + + + {/snippet} + + +
    + {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !schedules?.length} + {#if activeFilters} + + {:else} + scheduleEditor?.openNew(false), + aiId: 'schedules-empty-add', + aiDescription: 'Add schedule' + }} + /> + {/if} + {:else if items?.length} +
    + {#each items.slice(0, nbDisplayed) as { path, error, summary, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, extra_perms, canWrite, jobs, paused_until, labels, inherited_labels, draft_only, is_draft } (path)} + {@const hasDraft = + getLocalDraftHint($operatingWorkspace, 'trigger_schedule', path) ?? is_draft} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + {@const avg_s = jobs + ? jobs.reduce((acc, x) => acc + x.duration_ms, 0) / jobs.length + : undefined} + +
    +
    + + + { + handledHash = `#${path}` + editSchedule(path, is_flow) + }} + class="min-w-0 grow hover:underline decoration-gray-400" + > +
    + {summary || script_path}{hasDraft ? '*' : ''} +
    +
    + schedule: {path} +
    +     + {#if labels?.length} + {#each labels as label} + { + const arr = (filters.val.label ?? '').split(',').filter(Boolean) + const idx = arr.indexOf(label) + if (idx >= 0) arr.splice(idx, 1) + else arr.push(label) + const newFilters = { ...filters.val } + if (arr.length) newFilters.label = arr.join(',') + else delete newFilters.label + filters.val = newFilters + }}>{label} + {/each} + {/if} + + + {#if paused_until && new Date(paused_until) > new Date()} +
    + Paused until {new Date(paused_until).toLocaleString()} +
    + {/if} + + + + + +
    + {#if error} + + + + + + {#snippet text()} +
    + The schedule disabled itself because there was an error scheduling the + next job: {error} +
    + {/snippet} +     + {/if} +
    + +
    + + {#key toggleResetVersions[path] ?? 0} + { + if (canWrite) { + setScheduleEnabled(path, e.detail) + } else { + sendUserToast('not enough permission', true) + // Permission denied — bump the row's reset + // counter so the Toggle remounts back to the + // prop value. Without this, the local + // `bind:checked` flip from the user's click + // stays stuck on. + bumpToggleReset(path) + } + }} + /> + {/key} +
    +
    + + + { + if (hosted) hosted.openLink(`${base}${href}`) + else goto(href) + } + }, + { + displayName: `Duplicate schedule`, + icon: Copy, + action: () => { + scheduleEditor?.openNew(is_flow, script_path, path) + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: !canWrite, + action: async () => { + await ScheduleService.deleteSchedule({ + workspace: $operatingWorkspace ?? '', + path + }) + loadSchedules() + } + }, + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => { + editSchedule(path, is_flow) + } + }, + ...(isDeployable('trigger', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'trigger', { + triggers: { + kind: 'schedules' + } + }) + } + } + ] + : []), + { + displayName: 'View runs', + icon: List, + href: `${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true` + }, + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: 'Run now', + icon: Play, + action: () => { + runScheduleNow(script_path, path, is_flow, $operatingWorkspace!) + } + }, + { + displayName: 'Permissions', + icon: Shield, + action: () => { + shareModal?.openDrawer(path, 'schedule') + } + } + ]} + /> +
    +
    +
    + {#if loadingSchedulesWithJobStats} +
    + + Job stats loading... +
    + {:else} +
    + {#if avg_s} +
    Avg: {(avg_s / 1000).toFixed(2)}s
    + {/if} + {#each jobs ?? [] as job} + {@const h = (avg_s ? job.duration_ms / avg_s : 1) * 7 + 3} + + +
    +
    + +
    +     +     + {/each} +
    + {/if} +
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'at ' : ''}{displayDate(edited_at)}
    +
    + {/each} +
    + {:else} + + {/if} +
    + {#if items && items?.length > 15 && nbDisplayed < items.length} +
    + {nbDisplayed} items out of {items.length} + +
    + {/if} +     +{/if} + + { + loadSchedules() + }} +/> diff --git a/frontend/src/lib/components/schedules/schedulesFilter.ts b/frontend/src/lib/components/schedules/schedulesFilter.ts index 24fb1c90a3..4b887e5aa4 100644 --- a/frontend/src/lib/components/schedules/schedulesFilter.ts +++ b/frontend/src/lib/components/schedules/schedulesFilter.ts @@ -73,15 +73,14 @@ export function buildSchedulesFilterSchema({ icon: Tag, description: 'Filter by label (comma-separated for multiple)' }, - ...(showUserFoldersFilter - ? { - user_folders_only: { - type: 'boolean' as const, - label: userFoldersLabel || 'User folders only', - icon: Users, - description: 'Show only schedules in user folders' - } - } - : {}) + // The key stays for every role, hidden where the filter does not apply: the list's URL sync + // reads the key set once, before the acting user that gates it may have loaded. + user_folders_only: { + type: 'boolean' as const, + label: userFoldersLabel || 'User folders only', + icon: Users, + description: 'Show only schedules in user folders', + hidden: !showUserFoldersFilter + } } satisfies FilterSchemaRec } diff --git a/frontend/src/lib/components/sessions/PageListView.svelte b/frontend/src/lib/components/sessions/PageListView.svelte new file mode 100644 index 0000000000..7dab54c494 --- /dev/null +++ b/frontend/src/lib/components/sessions/PageListView.svelte @@ -0,0 +1,131 @@ + + +{#snippet loading()} +
    + +
    +{/snippet} + + +
    + {#key `${path}:${workspaceId}:${reloadNonce}`} + {#if listed?.kind === 'variable'} + {#await import('$lib/components/variables/VariablesList.svelte')} + {@render loading()} + {:then Module} + + {/await} + {:else if listed?.kind === 'resource'} + {#await import('$lib/components/resources/ResourcesList.svelte')} + {@render loading()} + {:then Module} + + {/await} + {:else if listed?.kind === 'schedule'} + {#await import('$lib/components/schedules/SchedulesList.svelte')} + {@render loading()} + {:then Module} + + {/await} + {:else if listed?.kind === 'trigger'} + {#await import('$lib/components/triggers/TriggerList.svelte')} + {@render loading()} + {:then Module} + + {/await} + {/if} + {/key} +
    diff --git a/frontend/src/lib/components/sessions/PreviewTabHost.svelte b/frontend/src/lib/components/sessions/PreviewTabHost.svelte index b88907bf02..f1d630caca 100644 --- a/frontend/src/lib/components/sessions/PreviewTabHost.svelte +++ b/frontend/src/lib/components/sessions/PreviewTabHost.svelte @@ -105,6 +105,7 @@ // A page item's editor reads its draft only when it loads, so a reload remounts it. let pageItemReloadNonce = $state(0) + let pageListReloadNonce = $state(0) export function reload() { // A live editor shares the runtime store the chat mutates, so generic chat @@ -116,6 +117,10 @@ pageItemReloadNonce++ return } + if (slot.kind === 'pagelist') { + pageListReloadNonce++ + return + } try { const win = frame?.contentWindow if (!win) return @@ -334,6 +339,27 @@ {/await} {/if}
    +{:else if slot.kind === 'pagelist' && mounted && runtime} +
    + {#if overlayHostEl} + {#await import('./PageListView.svelte')} + {@render editorLoading()} + {:then Module} + + {/await} + {/if} +
    {:else if slot.kind === 'artifact' && mounted}
    { - const { pathname, search, hash } = window.location - if (hash) history.replaceState(history.state, '', `${pathname}${search}`) - } - dropAnchor() - setTimeout(dropAnchor, 0) - return true -} - /** * Drop the row a list page deep-links, once its drawer closes. The hash is how the row was * requested; leaving it behind makes the location claim a drawer that is no longer open — diff --git a/frontend/src/lib/components/sessions/previewPaths.ts b/frontend/src/lib/components/sessions/previewPaths.ts index 13d5abca33..58eca9504d 100644 --- a/frontend/src/lib/components/sessions/previewPaths.ts +++ b/frontend/src/lib/components/sessions/previewPaths.ts @@ -79,6 +79,11 @@ export function pageItemListPath(ref: PageItemRef): string { } } +/** Whether a (base-stripped) path is the list page of a page item kind. */ +export function isPageItemListPath(path: string): boolean { + return pageItemForListPath(path, '') !== undefined +} + /** The page item a list page's row names, or undefined for a page that lists none. */ export function pageItemForListPath(pagePath: string, path: string): PageItemRef | undefined { const clean = stripBase(pagePath) diff --git a/frontend/src/lib/components/sessions/previewRouter.test.ts b/frontend/src/lib/components/sessions/previewRouter.test.ts index 3bc76882e5..143ce42975 100644 --- a/frontend/src/lib/components/sessions/previewRouter.test.ts +++ b/frontend/src/lib/components/sessions/previewRouter.test.ts @@ -284,6 +284,17 @@ describe('resolvePreviewTab', () => { expect(resolvePreviewTab('/runs')).toEqual({ kind: 'iframe' }) }) + it('mounts the list page of a page item kind in process, whatever its filters', () => { + expect(resolvePreviewTab('/variables?path_start=f%2Fa%2F')).toEqual({ + kind: 'pagelist', + path: '/variables' + }) + expect(resolvePreviewTab('/kafka_triggers')).toEqual({ + kind: 'pagelist', + path: '/kafka_triggers' + }) + }) + it('routes any script item to a live editor', () => { expect(resolvePreviewTab('/scripts/edit/f/foo/bar')).toEqual({ kind: 'editor', diff --git a/frontend/src/lib/components/sessions/previewRouter.ts b/frontend/src/lib/components/sessions/previewRouter.ts index 13e7626e70..de21660f3f 100644 --- a/frontend/src/lib/components/sessions/previewRouter.ts +++ b/frontend/src/lib/components/sessions/previewRouter.ts @@ -3,6 +3,7 @@ import { AUDIT_LOGS_PATH, FOLDERS_PATH, GROUPS_PATH, + isPageItemListPath, pageItemForListPath, pageItemListPath, pageItemPageHref, @@ -530,13 +531,14 @@ export const isArtifactKey = (key: string) => key.startsWith('artifact:') // fallback. Any editable item of a wrappable kind (script, flow, raw app) mounts // its per-(kind,path) cell editor; a `/pipeline/` route mounts the // data-pipeline graph editor (single, shared runtime.pipelineEditorState — `path` -// is the folder); everything else (static pages, regular drag-and-drop apps, any -// other route) stays an iframe. +// is the folder); the list page of a page item kind mounts that list; everything +// else (static pages, regular drag-and-drop apps, any other route) stays an iframe. export type PreviewSlot = | { kind: 'editor'; editorKind: SessionTargetKind | 'pipeline'; path: string } | { kind: 'artifact'; id: string; version?: number } | { kind: 'runform'; toolCallId: string } | { kind: 'pageitem'; ref: PageItemRef } + | { kind: 'pagelist'; path: string } | { kind: 'iframe' } export function resolvePreviewTab(url: string): PreviewSlot { @@ -546,22 +548,24 @@ export function resolvePreviewTab(url: string): PreviewSlot { if (runForm) return { kind: 'runform', toolCallId: runForm.toolCallId } const pageItem = parsePageItemRoute(url) if (pageItem) return { kind: 'pageitem', ref: pageItem } + const route = stripBase(url) + if (isPageItemListPath(route)) return { kind: 'pagelist', path: route } const pipelineFolder = parsePipelineRoute(url) if (pipelineFolder) { return { kind: 'editor', editorKind: 'pipeline', path: pipelineFolder } } - const route = parsePreviewItemRoute(url) - if (!route) return { kind: 'iframe' } + const item = parsePreviewItemRoute(url) + if (!item) return { kind: 'iframe' } const editorKind: SessionTargetKind | undefined = - route.kind === 'script' + item.kind === 'script' ? 'script' - : route.kind === 'flow' + : item.kind === 'flow' ? 'flow' - : route.kind === 'app' && route.raw_app + : item.kind === 'app' && item.raw_app ? 'raw_app' : undefined if (!editorKind) return { kind: 'iframe' } - return { kind: 'editor', editorKind, path: route.itemPath } + return { kind: 'editor', editorKind, path: item.itemPath } } /** The full workspace page showing what a tab shows ("Open in workspace"), or undefined when @@ -576,6 +580,7 @@ export function workspacePageHref(location: string): string | undefined { case 'pageitem': return pageItemPageHref(slot.ref) case 'editor': + case 'pagelist': case 'iframe': return location default: { diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index fbb84e1431..6edc8ab1dc 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -406,7 +406,8 @@ function createRuntime(session: Session): SessionRuntime { const tab = owner?.displayedTab if (!tab) return undefined const slotKind = resolvePreviewTab(tab.url).kind - if (slotKind !== 'iframe' && slotKind !== 'pageitem') return undefined + if (slotKind !== 'iframe' && slotKind !== 'pageitem' && slotKind !== 'pagelist') + return undefined return previewLocationContext(whereIs(tab)) } // Pre-flight: materialise the (still-transient) session, then commit diff --git a/frontend/src/lib/components/triggers/TriggerList.svelte b/frontend/src/lib/components/triggers/TriggerList.svelte new file mode 100644 index 0000000000..501eebd90f --- /dev/null +++ b/frontend/src/lib/components/triggers/TriggerList.svelte @@ -0,0 +1,983 @@ + + +{#snippet rowTitle(row: TriggerW & { marked?: any }, hasDraft: boolean | undefined)} + {@const marked = ['http', 'websocket', 'kafka', 'nats', 'email'].includes(triggerKind) + ? row.marked + : undefined} +
    + {#if marked} + + {@html marked} + + {:else if triggerKind === 'http'} + {#if row.summary} + {row.summary} + {:else} + {row.http_method!.toUpperCase()} + /{isCloudHosted() || row.workspaced_route || globalHttpWorkspacedRoute + ? row.workspace_id + '/' + row.route_path + : row.route_path} + {/if} + {:else if triggerKind === 'websocket'} + {websocketUrl(row.url!)} + {:else if triggerKind === 'kafka'} + {row.kafka_resource_path} - {row.topics!.join(', ')} + {:else if triggerKind === 'nats'} + {row.nats_resource_path} - {row.subjects!.join(', ')} + {:else if triggerKind === 'email'} + {emailAddress(row)} + {:else if triggerKind === 'gcp'} + {row.path} - {row.topic_id} + {:else if triggerKind === 'azure'} + {row.path} - {row.topic_name + ? row.topic_name + : (row.scope_resource_id?.split('/')?.pop() ?? '')} ({row.azure_mode}, {row.subscription_name}) + {:else} + {row.path} + {/if}{hasDraft ? '*' : ''} +
    + {#if ['http', 'websocket', 'kafka', 'nats', 'email'].includes(triggerKind)} +
    {row.path}
    + {:else if triggerKind === 'postgres'} +
    + {row.postgres_resource_path} +
    + {/if} +
    + {#if triggerKind === 'http' && row.static_asset_config} + file: {row.static_asset_config.s3} + {:else} + runnable: {row.script_path} + {/if} +
    +{/snippet} + +{#snippet statusDot(row: TriggerW)} + {@const status = config.status} + {#if status} + {@const ping = status.pings + ? row.last_server_ping + ? new Date(row.last_server_ping) + : undefined + : new Date()} + {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} + {@const effectiveMode = row.draft_only ? 'disabled' : row.mode} + {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} +
    + {#if (enabled && (!pinging || row.error)) || (!enabled && row.error) || (enabled && !row.server_id)} + + + + + + {#snippet text()} +
    + {#if enabled} + {#if !row.server_id} + {status.starting} + {:else} + {status.notConnected}{row.error ? ': ' + row.error : ''} + {/if} + {:else} + {status.disabled}: {row.error} + {/if} +
    + {/snippet} +     + {:else if enabled} + + + + + {#snippet text()} +
    + {status.connected}{status.shuttingDown && !row.server_id ? ' (shutting down...)' : ''} +
    + {/snippet} +     + {/if} +
    + {/if} +{/snippet} + + { + isDeleting = false + pendingDelete = undefined + }} + on:confirmed={confirmPendingDelete} +> +
    + Are you sure you want to remove this trigger? + {#if pendingDelete && triggerKind === 'postgres'} + + + {:else if pendingDelete && triggerKind === 'gcp'} + + {:else if pendingDelete?.subscription_name && triggerKind === 'azure'} + + {/if} +
    +     + + +{#await EDITORS[triggerKind]() then Editor} + +{/await} +{#if triggerKind === 'http'} + {#await import('./http/RoutesGenerator.svelte') then Generator} + + {/await} + {#await import('./http/OpenAPISpecGenerator.svelte') then Generator} + + {/await} +{/if} + + (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} +/> + +{#if operatorBlocked} + +{:else} + + +
    + {#if triggerKind === 'http'} + + + {/if} + {#if canCreate} + + {/if} +
    +     + + {#if config.cloudDisabled && isCloudHosted()} + + {config.title} are disabled in the multi-tenant cloud. + +
    + {/if} +
    +
    + +
    +
    Filter by path of
    + + {#snippet children({ item })} + + + {/snippet} + +
    + + +
    + {#if operatingUser.current?.is_super_admin && operatingUser.current.username.includes('@')} + + {:else if isAdmin} + + {/if} +
    +
    + {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !triggers?.length} + editor?.openNew(false), + aiId: config.empty.aiId, + aiDescription: config.empty.aiDescription + } + : undefined} + /> + {:else if items?.length} +
    + {#each items.slice(0, nbDisplayed) as row (row.path)} + {@const { + path, + edited_by, + edited_at, + script_path, + is_flow, + extra_perms, + canWrite, + mode, + retry, + error_handler_path, + error_handler_args, + labels, + draft_only, + is_draft + } = row} + {@const hasDraft = + getLocalDraftHint($operatingWorkspace, config.draftKind, path) ?? is_draft} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + {@const effectiveMode = draft_only ? 'disabled' : mode} + {@const live = isLive(row)} + {@const copy = copyUrl(row)} + +
    +
    + + + { + if (hosted) e.preventDefault() + openEdit(path, is_flow) + }} + class="min-w-0 grow hover:underline decoration-gray-400" + > + {@render rowTitle(row, hasDraft)} + + + + + {#if live} + {@render statusDot(row)} + {/if} + +
    + + {#if live} + onToggleMode(path, newMode)} + triggerMode={effectiveMode} + includeModalConfig={{ + triggerPath: path, + triggerKind, + runnableConfig: { + path: script_path, + kind: is_flow ? 'flow' : 'script', + retry, + errorHandlerPath: error_handler_path, + errorHandlerArgs: error_handler_args + } + }} + {canWrite} + hideToggleLabels + hideDropdown + /> + {/if} +
    + +
    + {#if copy} + + {/if} + + openLink(href) + }, + ...(canWrite && !draft_only && mode !== 'suspended' + ? [ + { + displayName: 'Suspend job execution', + icon: Pause, + action: () => { + onToggleMode(path, 'suspended') + } + } + ] + : []), + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => openEdit(path, is_flow) + }, + ...(isDeployable('trigger', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'trigger', { + triggers: { kind: config.deployKind } + }) + } + } + ] + : []), + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: 'Permissions', + icon: Shield, + action: () => { + shareModal?.openDrawer(path, config.shareKind as any) + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: !canWrite || (config.adminOnly && !isAdmin), + action: () => deleteTrigger(row) + } + ]} + /> +
    +
    +
    +
    + {#if edited_by}
    edited by {edited_by}
    {/if} +
    {edited_by ? 'at ' : ''}{displayDate(edited_at)}
    +
    +
    +
    + {/each} +
    + {:else} + + {/if} +
    + {#if items && items?.length > 15 && nbDisplayed < items.length} + {nbDisplayed} items out of {items.length} + + {/if} +     +{/if} + + { + loadTriggers() + }} +/> diff --git a/frontend/src/lib/components/triggers/amqp/AmqpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/amqp/AmqpTriggerEditorInner.svelte index 7e3c2c5ac1..3afcdf1a43 100644 --- a/frontend/src/lib/components/triggers/amqp/AmqpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/amqp/AmqpTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { untrack } from 'svelte' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -176,7 +175,6 @@ defaultConfig?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.amqp.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() let loadingTimeout = setTimeout(() => { diff --git a/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte index 697bc00ad4..12b6bf0533 100644 --- a/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -156,7 +155,6 @@ isFlow: boolean, defaultValues?: Record ) { - if (handOffPageDrawer(TRIGGER_PAGES.azure.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() drawerLoading = true diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte index 30f848004f..fe472cb8c4 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -143,7 +142,6 @@ defaultConfig?: Partial, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.email.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() drawerLoading = true diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index 92902aa44b..03c4a12261 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -167,7 +166,6 @@ defaultValues?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.gcp.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() drawerLoading = true diff --git a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte index 2b28ae0d8d..5fc2999704 100644 --- a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte @@ -2,7 +2,6 @@ import { Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -274,7 +273,6 @@ isFlow: boolean, defaultConfig?: Partial ) { - if (handOffPageDrawer(TRIGGER_PAGES.http.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() drawerLoading = true diff --git a/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte b/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte index 91b2ee7e9e..71b54a39e0 100644 --- a/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte +++ b/frontend/src/lib/components/triggers/http/RoutesGenerator.svelte @@ -16,16 +16,20 @@ import RouteEditor from './RouteEditor.svelte' import { generateHttpTriggerFromOpenApi, type Source } from './utils' import { isCloudHosted } from '$lib/cloud' - import { usedTriggerKinds, userStore } from '$lib/stores' + import { usedTriggerKinds } from '$lib/stores' import FileInput from '../../common/fileInput/FileInput.svelte' import { emptyStringTrimmed, sendUserToast } from '$lib/utils' import FolderPicker from '../../FolderPicker.svelte' import Required from '$lib/components/Required.svelte' import { Drawer, DrawerContent } from '$lib/components/common' import { get } from 'svelte/store' - import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() type Props = { closeFn: () => Promise @@ -137,7 +141,9 @@ } } - let userIsAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) + let userIsAdmin = $derived( + operatingUser.current?.is_admin || operatingUser.current?.is_super_admin + ) async function generateHttpTrigger() { try { diff --git a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte index 3fb5668f44..ea604bc5a8 100644 --- a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -178,7 +177,6 @@ defaultConfig?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.kafka.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() let loadingTimeout = setTimeout(() => { diff --git a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte index f3a42f0d49..3627bd2d4d 100644 --- a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { untrack } from 'svelte' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -173,7 +172,6 @@ defaultConfig?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.mqtt.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() let loadingTimeout = setTimeout(() => { diff --git a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte index 16997be11a..045f7155d3 100644 --- a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -161,7 +160,6 @@ defaultConfig?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.nats.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() let loadingTimeout = setTimeout(() => { diff --git a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte index 23ce3fc43c..546b4d2457 100644 --- a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Button, TabContent } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -262,7 +261,6 @@ defaultConfig?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.postgres.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() let loadingTimeout = setTimeout(() => { diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index a6a6b5e7a8..f0eed8a9fe 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Badge, Button, ButtonType, Tab, Tabs } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { SCHEDULES_PATH } from '$lib/components/sessions/previewPaths' @@ -181,7 +180,6 @@ defaultCfg?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(SCHEDULES_PATH, ePath)) return let loadingTimeout = setTimeout(() => { showLoading = true }, 100) // Do not show loading spinner for the first 100ms diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte index 0b3f846c97..5521eca4c3 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte @@ -2,7 +2,6 @@ import { Alert, Button } from '$lib/components/common' import { clearPageDrawerAnchor, - handOffPageDrawer, setPageDrawerAnchor } from '$lib/components/sessions/pageDrawerSession' import { TRIGGER_PAGES } from '$lib/components/sessions/previewPaths' @@ -156,7 +155,6 @@ defaultConfig?: Record, fixedScriptPath_?: string ) { - if (handOffPageDrawer(TRIGGER_PAGES.sqs.path, ePath)) return // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() let loadingTimeout = setTimeout(() => { diff --git a/frontend/src/lib/components/triggers/triggerListConfig.ts b/frontend/src/lib/components/triggers/triggerListConfig.ts new file mode 100644 index 0000000000..9e5b24c8f2 --- /dev/null +++ b/frontend/src/lib/components/triggers/triggerListConfig.ts @@ -0,0 +1,506 @@ +import { + AmqpTriggerService, + AzureTriggerService, + EmailTriggerService, + GcpTriggerService, + HttpTriggerService, + KafkaTriggerService, + MqttTriggerService, + NatsTriggerService, + PostgresTriggerService, + SqsTriggerService, + WebsocketTriggerService, + type AmqpTrigger, + type AzureTrigger, + type EmailTrigger, + type GcpTrigger, + type HttpTrigger, + type KafkaTrigger, + type MqttTrigger, + type NatsTrigger, + type PostgresTrigger, + type SqsTrigger, + type TriggerMode, + type UserDraftItemKind, + type WebsocketTrigger +} from '$lib/gen' +import type { TriggerKind as UsedTriggerKind } from '$lib/components/triggers' +import type { TriggerKind } from '$lib/components/sessions/previewPaths' + +type AnyTrigger = + | HttpTrigger + | WebsocketTrigger + | PostgresTrigger + | KafkaTrigger + | NatsTrigger + | MqttTrigger + | AmqpTrigger + | SqsTrigger + | GcpTrigger + | AzureTrigger + | EmailTrigger +type KeysOfUnion = T extends unknown ? keyof T : never +type FieldOf = T extends unknown + ? K extends keyof T + ? T[K] + : never + : never + +/** A row of any trigger list: the fields every kind shares, plus every kind's own as optional, + * so a field read by one kind's branch is still checked against the generated types. */ +export type TriggerRow = Pick< + HttpTrigger, + 'path' | 'script_path' | 'is_flow' | 'mode' | 'edited_by' | 'edited_at' +> & { + [K in Exclude< + KeysOfUnion, + 'path' | 'script_path' | 'is_flow' | 'mode' | 'edited_by' | 'edited_at' + >]?: FieldOf +} + +type Call = (a: A) => Promise + +export type TriggerListConfig = { + /** Page title, and the kind's plural name in the cloud alert. */ + title: string + tooltip: string + documentationLink?: string + /** Follows "New" on the create button. */ + newLabel: string + searchPlaceholder: string + /** The "Filter by path of" choice naming the trigger itself. */ + filterLabel: string + empty: { + title: string + description: string + actionLabel: string + aiId: string + aiDescription: string + } + /** Disabled in the multi-tenant cloud. */ + cloudDisabled: boolean + /** Hidden from operators unless the workspace grants them triggers. */ + operatorGate: boolean + /** Creating and deleting are for admins only. */ + adminOnly: boolean + usedKind: UsedTriggerKind + deployKind: UsedTriggerKind + shareKind: string + draftKind: UserDraftItemKind + /** Kinds holding a live connection: polled, with a status dot. Each message is followed by + * the error or state it reports. */ + status?: { + starting: string + notConnected: string + disabled: string + connected: string + /** Reports a connection whose server is going away. */ + shuttingDown: boolean + /** The server stamps a ping; without one, a connected server counts as alive. */ + pings: boolean + } + /** Mode changes run through the fork-conflict check, under this name. Kinds whose runtime + * address is workspace-prefixed skip it: a fork and its parent never collide. */ + forkConflictLabel?: string + /** Toast a mode change that went through, as "
    + {#if !filteredItems} + + {#each new Array(3) as _} + + {/each} + {:else if filteredItems.length == 0} + {#if activeFilters} + + {:else} + variableEditor?.initNew(), + aiId: 'variables-empty-add-variable', + aiDescription: 'Add variable' + } + : undefined} + /> + {/if} + {:else} + + + + + Path + Value + Description + Status + Actions + + + + {#each filteredItems as { path, value, is_secret, description, extra_perms, canWrite, account, is_refreshed, is_expired, refresh_error, is_linked, labels, inherited_labels, ws_specific, draft_only, is_draft }} + {@const hasDraft = + getLocalDraftHint($operatingWorkspace, 'variable', path) ?? is_draft} + + + + + +
    + { + handledHash = `#${path}` + editVariable(path) + }} + href="#{path}" + > + {path}{hasDraft ? '*' : ''} + + + {#if labels?.length} + {#each labels as label} + { + const arr = (filters.val.label ?? '').split(',').filter(Boolean) + const idx = arr.indexOf(label) + if (idx >= 0) arr.splice(idx, 1) + else arr.push(label) + const newFilters = { ...filters.val } + if (arr.length) newFilters.label = arr.join(',') + else delete newFilters.label + filters.val = newFilters + }}>{label} + {/each} + {/if} + +
    + + + +
    + {#if value} + {truncate(value, 20)} + {:else} + ∗∗∗∗ + {/if} +
    + {#if is_secret} + + + {#snippet text()} + This item is secret + {/snippet} + + {/if} +     +     + + {truncate(description ?? '', 50)} + + + +
    + {#if is_linked} + + + {#snippet text()} +
    + This variable is linked with a resource of the same path. They are + deleted and renamed together. +
    + {/snippet} +     + {/if} + {#if account} + + + {#snippet text()} +
    + This OAuth token will be kept up-to-date in the background by + Windmill using its refresh token +
    + {/snippet} +     + {/if} + + {#if is_refreshed} +
    + {#if refresh_error} + + +
    + + +
    + + {#snippet text()} +
    + Latest exchange of the refresh token did not succeed. Error: {refresh_error} +
    + {/snippet} +     + {:else if is_expired} + + + {#snippet text()} +
    + The access_token is expired, it will get renewed the next time + this variable is fetched or you can request is to be refreshed + in the dropdown on the right. +
    + {/snippet} +     + {:else} + + + {#snippet text()} +
    + The variable was connected through OAuth and the token is not + expired. +
    + {/snippet} +     + {/if} +
    + {/if} +
    +     + + { + let owner = isOwner(path, operatingUser.current, $operatingWorkspace) + return [ + { + displayName: 'Edit', + icon: Pen, + action: () => editVariable(path), + disabled: !canWrite || !showCreateButtons + }, + { + displayName: 'Delete', + icon: Trash, + type: 'delete', + action: (event) => { + if (event['shiftKey']) { + deleteVariable(path, account) + } else { + deleteIsLinked = is_linked ?? false + deleteConfirmedCallback = () => { + deleteVariable(path, account) + } + } + }, + disabled: !owner || !showCreateButtons + }, + ...(!ws_specific && + isDeployable(is_secret ? 'secret' : 'variable', path, deployUiSettings) + ? [ + { + displayName: 'Deploy to prod/staging', + icon: FileUp, + action: () => { + deploymentDrawer?.openDrawer(path, 'variable') + } + } + ] + : []), + { + displayName: 'Permissions', + action: () => { + shareModal?.openDrawer(path, 'variable') + }, + icon: Shield + }, + ...(account != undefined + ? [ + { + displayName: 'Refresh token', + icon: RefreshCw, + action: async () => { + await OauthService.refreshToken({ + workspace: $operatingWorkspace ?? '', + id: account ?? 0, + requestBody: { + path + } + }) + sendUserToast('Token refreshed') + loadVariables() + } + } + ] + : []) + ] + }} + /> + + + {/each} + + + {/if} +
    + {:else if tab == 'contextual'} +
    + {#if loading.contextual} + + {#each new Array(8) as _} + + {/each} + {:else} + + {#if contextualVariables.filter((x) => x.is_custom).length === 0} + contextualVariableEditor?.initNew(), + aiId: 'variables-empty-add-contextual-variable', + aiDescription: 'Add contextual variable' + } + : undefined} + /> + {:else} + x.is_custom)} + keys={['name', 'value']} + getRowActions={operatingUser.current?.is_admin || + operatingUser.current?.is_super_admin + ? (row) => { + return [ + { + displayName: 'Edit', + action: () => contextualVariableEditor?.editVariable(row.name, row.value) + }, + { + displayName: 'Delete', + type: 'delete', + action: () => { + deleteContextualVariable(row) + } + } + ] + } + : undefined} + /> + {/if} + + !x.is_custom)} + keys={['name', 'value', 'description']} + /> + {/if} +
    + {/if} + + +{/if} + + { + deleteConfirmedCallback = undefined + }} + on:confirmed={() => { + if (deleteConfirmedCallback) { + deleteConfirmedCallback() + } + deleteConfirmedCallback = undefined + }} +> +
    + Are you sure you want to remove this variable? + {#if deleteIsLinked} + + This variable is linked with a resource of the same path. The linked resource will also be + deleted. + + {/if} + +
    + You can press + SHIFT + while removing a variable to bypass confirmation. +
    +     +
    +     diff --git a/frontend/src/lib/components/variables/variablesFilter.ts b/frontend/src/lib/components/variables/variablesFilter.ts index 5f7ce1e51a..d609182da0 100644 --- a/frontend/src/lib/components/variables/variablesFilter.ts +++ b/frontend/src/lib/components/variables/variablesFilter.ts @@ -64,15 +64,14 @@ export function buildVariablesFilterSchema({ icon: Tag, description: 'Filter by label (comma-separated for multiple)' }, - ...(showUserFoldersFilter - ? { - user_folders_only: { - type: 'boolean' as const, - label: userFoldersLabel || 'User folders only', - icon: Users, - description: 'Show only variables in user folders' - } - } - : {}) + // The key stays for every role, hidden where the filter does not apply: the list's URL sync + // reads the key set once, before the acting user that gates it may have loaded. + user_folders_only: { + type: 'boolean' as const, + label: userFoldersLabel || 'User folders only', + icon: Users, + description: 'Show only variables in user folders', + hidden: !showUserFoldersFilter + } } satisfies FilterSchemaRec } diff --git a/frontend/src/lib/svelte5UtilsKit.svelte.ts b/frontend/src/lib/svelte5UtilsKit.svelte.ts index 7b6420ab7e..d77b96df06 100644 --- a/frontend/src/lib/svelte5UtilsKit.svelte.ts +++ b/frontend/src/lib/svelte5UtilsKit.svelte.ts @@ -3,6 +3,8 @@ import { z } from 'zod' import { serializeParam } from '$lib/utils/serializeParam' +import { untrack } from 'svelte' +import { useHostedPage } from '$lib/components/hostedPage' export type SearchParamsResult = S extends z.ZodObject @@ -58,12 +60,14 @@ function deserializeParam(raw: string, fieldSchema: z.ZodType): unknown { export function useSearchParams(schema: S): SearchParamsResult { const shape: Record = (schema as any).shape ?? {} const keys = Object.keys(shape) + const hosted = useHostedPage() + const currentSearch = () => (hosted ? hosted.search : window.location.search) // Reactive snapshot of search params - one $state cell per key const values: Record = $state( Object.fromEntries( keys.map((k) => { - const raw = new URLSearchParams(window.location.search).get(k) + const raw = new URLSearchParams(untrack(currentSearch)).get(k) const parsed = raw != null ? deserializeParam(raw, shape[k]) : null return [k, parsed] }) @@ -71,7 +75,7 @@ export function useSearchParams(schema: S): SearchParamsRes ) function syncFromUrl() { - const sp = new URLSearchParams(window.location.search) + const sp = new URLSearchParams(currentSearch()) for (const k of keys) { const raw = sp.get(k) const parsed = raw != null ? deserializeParam(raw, shape[k]) : null @@ -79,8 +83,13 @@ export function useSearchParams(schema: S): SearchParamsRes } } - // Keep in sync when the user navigates back/forward + // Keep in sync when the user navigates back/forward, or the host re-points the page $effect(() => { + if (hosted) { + hosted.search + untrack(syncFromUrl) + return + } window.addEventListener('popstate', syncFromUrl) return () => window.removeEventListener('popstate', syncFromUrl) }) @@ -94,12 +103,16 @@ export function useSearchParams(schema: S): SearchParamsRes }, set(v: unknown) { ;(values as any)[key] = v - const sp = new URLSearchParams(window.location.search) + const sp = new URLSearchParams(untrack(currentSearch)) if (v == null) { sp.delete(key) } else { sp.set(key, serializeParam(v)) } + if (hosted) { + hosted.setSearch(sp.size ? `?${sp}` : '') + return + } const hash = window.location.hash const newUrl = sp.toString() ? `${window.location.pathname}?${sp}${hash}` diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index bfae9bb5d2..e9d4493e66 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -90,7 +90,11 @@ import SessionModeSwitch from '$lib/components/sessions/SessionModeSwitch.svelte' import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' import { copilotInfo } from '$lib/aiStore' - import { parsePreviewItemRoute } from '$lib/components/sessions/previewPaths' + import { + isPageItemListPath, + parsePreviewItemRoute, + stripBase + } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' import { restoreSessionBackups } from '$lib/components/sessions/sessionMirror.svelte' @@ -431,6 +435,21 @@ } catch {} return } + // A list page mounts in process too, where its rows open as tabs of their own. + const listUrl = navigation.to?.url + if (listUrl && isPageItemListPath(stripBase(listUrl.pathname))) { + navigation.cancel() + const u = new URL(listUrl.href) + u.searchParams.delete('nomenubar') + u.searchParams.delete('workspace') + try { + window.parent.postMessage( + { type: 'wm.session.openList', href: u.pathname + u.search + u.hash }, + window.location.origin + ) + } catch {} + return + } const runTarget = previewRunTarget(navigation.to?.url) if (runTarget) { navigation.cancel() diff --git a/frontend/src/routes/(root)/(logged)/amqp_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/amqp_triggers/+page.svelte index 360defa995..8a21a3a61f 100644 --- a/frontend/src/routes/(root)/(logged)/amqp_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/amqp_triggers/+page.svelte @@ -1,570 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - AMQP triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - amqpTriggerEditor?.openNew(false), - aiId: 'amqp-triggers-empty-add', - aiDescription: 'Add AMQP trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, is_flow, extra_perms, canWrite, error, last_server_ping, server_id, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_amqp', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = last_server_ping ? new Date(last_server_ping) : undefined} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - amqpTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {path}{hasDraft ? '*' : ''} -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - AMQP trigger is starting... - {:else} - AMQP trigger is not connected{error ? ': ' + error : ''} - {/if} - {:else} - AMQP trigger was disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    - AMQP trigger is connected{!server_id ? ' (shutting down...)' : ''} -
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'amqp', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - amqpTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'amqp' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'amqp_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - await AmqpTriggerService.deleteAmqpTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/azure_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/azure_triggers/+page.svelte index 11afa9f94b..2193288ccf 100644 --- a/frontend/src/routes/(root)/(logged)/azure_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/azure_triggers/+page.svelte @@ -1,668 +1,5 @@ - { - deleteAzureTriggerCallback = undefined - }} - on:confirmed={async () => { - if (deleteAzureTriggerCallback) { - isDeleting = true - if (deleteSubscription && subscriptionDeletePayload) { - try { - const msg = await AzureTriggerService.deleteAzureSubscription({ - workspace: $workspaceStore ?? '', - path: subscriptionDeletePayload.azure_resource_path, - requestBody: { - azure_mode: subscriptionDeletePayload.azure_mode, - scope_resource_id: subscriptionDeletePayload.scope_resource_id, - topic_name: subscriptionDeletePayload.topic_name, - subscription_name: subscriptionDeletePayload.subscription_name - } - }) - sendUserToast(msg) - } catch (error) { - isDeleting = false - sendUserToast(error.body || error.message, true) - return - } - } - await deleteAzureTriggerCallback() - } - deleteAzureTriggerCallback = undefined - subscriptionToDelete = undefined - subscriptionDeletePayload = undefined - }} -> -
    - Are you sure you want to remove this trigger? - - {#if subscriptionToDelete} - - {/if} -
    -     - - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - Azure Event Grid triggers are disabled in the multi-tenant cloud. - - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - azureTriggerEditor?.openNew(false), - aiId: 'azure-triggers-empty-add', - aiDescription: 'Add Azure Event Grid trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { azure_resource_path, azure_mode, scope_resource_id, topic_name, subscription_name, workspace_id, path, edited_by, error, edited_at, script_path, is_flow, extra_perms, canWrite, mode, server_id, retry, error_handler_path, error_handler_args, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_azure', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = new Date()} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - {@const is_pull = azure_mode === 'namespace_pull'} - {@const topic_label = topic_name - ? topic_name - : (scope_resource_id?.split('/')?.pop() ?? '')} - -
    -
    - - - azureTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {path} - {topic_label} ({azure_mode}, {subscription_name}){hasDraft ? '*' : ''} -
    -
    - runnable: {script_path} -
    -     - - - - {#if is_pull} -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - Azure Event Grid trigger is starting... - {:else} - Could not connect to Azure Event Grid{error ? ': ' + error : ''} - {/if} - {:else} - Disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    - Connected to Azure Event Grid{!server_id - ? ' (shutting down...)' - : ''}
    - {/snippet} -     - {/if} -
    - {/if} - -
    - - {#if is_pull} - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'azure', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> - {/if} -
    - -
    - {#if !is_pull} - - {/if} - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - azureTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'azure' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'azure_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - isDeleting = false - deleteSubscription = false - subscriptionToDelete = subscription_name - subscriptionDeletePayload = { - azure_mode, - scope_resource_id, - topic_name: topic_name ?? undefined, - subscription_name, - azure_resource_path - } - deleteAzureTriggerCallback = async () => { - const message = await AzureTriggerService.deleteAzureTrigger({ - workspace: $workspaceStore ?? '', - path - }) - sendUserToast(message) - loadTriggers() - } - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/email_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/email_triggers/+page.svelte index 87c9558dbb..8fb28777c8 100644 --- a/frontend/src/routes/(root)/(logged)/email_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/email_triggers/+page.svelte @@ -1,553 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.triggers} - -{:else} - - - {#if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} - -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - emailTriggerEditor?.openNew(false), - aiId: 'email-triggers-empty-add', - aiDescription: 'Add email trigger' - } - : undefined} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { workspace_id, workspaced_local_part, path, edited_by, edited_at, script_path, is_flow, extra_perms, canWrite, marked, local_part, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'trigger_email', path) ?? is_draft} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const emailAddress = getEmailAddress( - local_part, - workspaced_local_part ?? false, - workspace_id, - emailDomain ?? '' - )} - -
    -
    - - - emailTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {#if marked} - - {@html marked} - - {:else} - {emailAddress} - {/if}{hasDraft ? '*' : ''} -
    -
    - {path} -
    -
    - runnable: {script_path} -
    -     - - - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'email', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - emailTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'routes' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'email_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: - !canWrite || !($userStore?.is_admin || $userStore?.is_super_admin), - action: async () => { - try { - await EmailTriggerService.deleteEmailTrigger({ - workspace: $workspaceStore ?? '', - path - }) - sendUserToast(`Successfully deleted Email trigger: ${path}`) - loadTriggers() - } catch (error) { - sendUserToast(error.body || error.message, true) - } - } - } - ]} - /> -
    -
    -
    -
    - {#if edited_by}
    edited by {edited_by}
    {/if} -
    {edited_by ? 'at ' : ''}{displayDate(edited_at)}
    -
    -
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     -{/if} - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte index 24c7553378..3b61eb7801 100644 --- a/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/gcp_triggers/+page.svelte @@ -1,654 +1,5 @@ - { - deleteGcpTriggerCallback = undefined - }} - on:confirmed={async () => { - if (deleteGcpTriggerCallback) { - isDeleting = true - if (deleteSubscription && deleteSubscriptionCallback) { - try { - await deleteSubscriptionCallback() - } catch (error) { - isDeleting = false - sendUserToast(error.body || error.message, true) - return - } - } - await deleteGcpTriggerCallback() - } - deleteGcpTriggerCallback = undefined - }} -> -
    - Are you sure you want to remove this trigger? - - -
    -     - - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - GCP Pub/Sub triggers are disabled in the multi-tenant cloud. - - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - gcpTriggerEditor?.openNew(false), - aiId: 'gcp-triggers-empty-add', - aiDescription: 'Add GCP Pub/Sub trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { gcp_resource_path, project_id, topic_id, workspace_id, delivery_type, path, edited_by, error, edited_at, script_path, is_flow, extra_perms, canWrite, mode, server_id, subscription_id, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_gcp', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = new Date()} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - gcpTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {path} - {topic_id}{hasDraft ? '*' : ''} -
    -
    - runnable: {script_path} -
    -     - - - - {#if delivery_type !== 'push'} -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - GCP Pub/Sub trigger is starting... - {:else} - Could not connect to GCP Pub/Sub{error ? ': ' + error : ''} - {/if} - {:else} - Disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    - Connected to GCP Pub/Sub{!server_id ? ' (shutting down...)' : ''}
    - {/snippet} -     - {/if} -
    - {/if} - -
    - - {#if delivery_type !== 'push'} - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'gcp', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> - {/if} -
    - -
    - {#if delivery_type === 'push'} - - {/if} - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - gcpTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'gcp' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'gcp_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - isDeleting = false - deleteSubscription = false - subscriptionToDelete = subscription_id - currentTopic = topic_id - deleteSubscriptionCallback = async () => { - const requestBody = { subscription_id, project_id } - const message = gcp_resource_path - ? await GcpTriggerService.deleteGcpSubscription({ - workspace: $workspaceStore ?? '', - path: gcp_resource_path, - requestBody - }) - : await GcpTriggerService.deleteGcpSubscriptionWithDefaultCredentials({ - workspace: $workspaceStore ?? '', - requestBody - }) - sendUserToast(message) - } - deleteGcpTriggerCallback = async () => { - const message = await GcpTriggerService.deleteGcpTrigger({ - workspace: $workspaceStore ?? '', - path - }) - sendUserToast(message) - loadTriggers() - } - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte index 6ee790818a..c4d4f904f6 100644 --- a/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte @@ -1,595 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.triggers} - -{:else} - - - - - - {#if isCloudHosted()} - - Kafka triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - kafkaTriggerEditor?.openNew(false), - aiId: 'kafka-triggers-empty-add', - aiDescription: 'Add Kafka trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, is_flow, kafka_resource_path, topics, extra_perms, canWrite, marked, server_id, error, last_server_ping, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'trigger_kafka', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = last_server_ping ? new Date(last_server_ping) : undefined} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - kafkaTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {#if marked} - - {@html marked} - - {:else} - {kafka_resource_path} - {topics.join(', ')} - {/if}{hasDraft ? '*' : ''} -
    -
    - {path} -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - Consumer is starting... - {:else} - Consumer is not connected{error ? ': ' + error : ''} - {/if} - {:else} - Consumer was disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    Consumer is connected
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'kafka', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - await KafkaTriggerService.deleteKafkaTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } - }, - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - kafkaTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'kafka' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'kafka_trigger') - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    Edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     -{/if} - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte index 126dfda031..41c30157ff 100644 --- a/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/mqtt_triggers/+page.svelte @@ -1,570 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - MQTT triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - mqttTriggerEditor?.openNew(false), - aiId: 'mqtt-triggers-empty-add', - aiDescription: 'Add MQTT trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, is_flow, extra_perms, canWrite, error, last_server_ping, server_id, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_mqtt', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = last_server_ping ? new Date(last_server_ping) : undefined} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - mqttTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {path}{hasDraft ? '*' : ''} -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - MQTT trigger is starting... - {:else} - MQTT trigger is not connected{error ? ': ' + error : ''} - {/if} - {:else} - MQTT trigger was disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    - MQTT trigger is connected{!server_id ? ' (shutting down...)' : ''} -
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'mqtt', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - mqttTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'mqtt' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'mqtt_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - await MqttTriggerService.deleteMqttTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte index 834869891a..ae96a52cfc 100644 --- a/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/nats_triggers/+page.svelte @@ -1,592 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.triggers} - -{:else} - - - - - - {#if isCloudHosted()} - - NATS triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - natsTriggerEditor?.openNew(false), - aiId: 'nats-triggers-empty-add', - aiDescription: 'Add NATS trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, is_flow, nats_resource_path, subjects, extra_perms, canWrite, marked, server_id, error, last_server_ping, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_nats', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = last_server_ping ? new Date(last_server_ping) : undefined} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - natsTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {#if marked} - - {@html marked} - - {:else} - {nats_resource_path} - {subjects.join(', ')} - {/if}{hasDraft ? '*' : ''} -
    -
    - {path} -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - Consumer is starting... - {:else} - Consumer is not connected{error ? ': ' + error : ''} - {/if} - {:else} - Consumer was disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    Consumer is connected
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'nats', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - natsTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'nats' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'nats_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - await NatsTriggerService.deleteNatsTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     -{/if} - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte index d1c07a7e75..a25078abe1 100644 --- a/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte @@ -1,674 +1,5 @@ - { - isDeleting = false - deletePostgresTriggerCallback = undefined - }} - on:confirmed={async () => { - if (deletePostgresTriggerCallback) { - let msg: string - isDeleting = true - if (deleteReplicationSlot && deleteReplicationSlotCallback) { - try { - msg = await deleteReplicationSlotCallback() - sendUserToast(msg) - } catch (error) { - isDeleting = false - sendUserToast(error.body || error.message, true) - return - } - } - if (deletePublication && deletePublicationCallback) { - try { - msg = await deletePublicationCallback() - sendUserToast(msg) - } catch (error) { - isDeleting = false - sendUserToast(error.body || error.message, true) - return - } - } - msg = await deletePostgresTriggerCallback() - sendUserToast(msg) - } - deletePostgresTriggerCallback = undefined - }} -> -
    - Are you sure you want to remove this trigger? - - - - -
    -     - - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - Postgres triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - postgresTriggerEditor?.openNew(false), - aiId: 'postgres-triggers-empty-add', - aiDescription: 'Add Postgres trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { postgres_resource_path, publication_name, replication_slot_name, path, edited_by, error, edited_at, script_path, is_flow, extra_perms, canWrite, mode, server_id, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'trigger_postgres', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = new Date()} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - postgresTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {path}{hasDraft ? '*' : ''} -
    -
    - {postgres_resource_path} -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - Postgres trigger is starting... - {:else} - Could not connect to database{error ? ': ' + error : ''} - {/if} - {:else} - Disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    Connected to database{!server_id ? ' (shutting down...)' : ''}
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'postgres', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - publicationToDelete = publication_name - replicationSlotToDelete = replication_slot_name - isDeleting = false - deletePublication = false - deleteReplicationSlot = false - deletePublicationCallback = async () => { - return await PostgresTriggerService.deletePostgresPublication({ - workspace: $workspaceStore!, - path: postgres_resource_path, - publication: publication_name - }) - } - - deleteReplicationSlotCallback = async () => { - return await PostgresTriggerService.deletePostgresReplicationSlot({ - workspace: $workspaceStore!, - path: postgres_resource_path, - requestBody: { - name: replication_slot_name - } - }) - } - - deletePostgresTriggerCallback = async () => { - const msg = await PostgresTriggerService.deletePostgresTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - return msg - } - } - }, - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - postgresTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'postgres' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'postgres_trigger') - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index bf62818170..8973bf83b4 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -1,1588 +1,5 @@ - { - deleteConfirmedCallback = undefined - }} - on:confirmed={() => { - if (deleteConfirmedCallback) { - deleteConfirmedCallback() - } - deleteConfirmedCallback = undefined - }} -> -
    - Are you sure you want to remove {deletePath}? - {#if deleteIsLinked} - - This resource is linked with a variable of the same path. That variable is deleted with it, - unless another resource still references it. - - {/if} - -
    - You can press - SHIFT - while removing a resource to bypass confirmation. -
    -     -
    -     - - - - - inferrer?.toggleDrawer?.()}> - - {#snippet actions()} - - {/snippet} - - - - - - {#snippet titleExtra()} - - {/snippet} -
    - {#if resourceTypeViewerObj.description} - - {/if} - {#if resourceTypeViewerObj.isFileset} - - This resource type represents a collection of files. Each file is identified by its - relative path and contains text content. In the CLI, filesets are stored as directories. - - {:else if resourceTypeViewerObj.formatExtension} - - This resource type represents a plain text file with a .{resourceTypeViewerObj.formatExtension} - extension. (e.g. my_file.{resourceTypeViewerObj.formatExtension}) - - {:else} - - {/if} -
    -     -     - - - - {#snippet actions()} - - {/snippet} -
    - - -
    - {#if editResourceType.isFileset} - - This resource type represents a collection of files. The schema cannot be edited. - - {:else if editResourceType.formatExtension} - - This resource type represents a plain text file with a .{editResourceType.formatExtension} - extension. (e.g. my_file.{editResourceType.formatExtension}). The schema only - contains a `content` field and thus cannot be edited. - - {:else} -
    Schema
    -
    - -
    - {/if} -
    -
    -     -     - - - - {#snippet actions()} - - {/snippet} -
    - - -
    -
    -
    Schema
    -
    - -
    -
    - -
    - -
    -
    -
    -     -     - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.resources} - -{:else} - - - {#if showCreateButtons} -
    - - -
    - {/if} -     - (showCreateButtons = v)} /> -
    - { - if (e.detail == 'cache') { - loading.resources = true - loadCache() - } else if (e.detail == 'states') { - loading.resources = true - loadState() - } else if (e.detail == 'theme') { - loading.resources = true - loadTheme() - } - }} - > - - - {#snippet extra()} - - Every resource has a Resource Type attached to it which contains its schema and make - it easy in scripts and flows to accept only resources of a specific resource type. - - {/snippet} - - - {#snippet extra()} - - States are actually resources (but excluded from the Workspace tab for clarity). - States are used by scripts to keep data persistent between runs of the same script by - the same trigger (schedule or user) - - {/snippet} - - - {#snippet extra()} - - Cached results are actually resources (but excluded from the Workspace tab for - clarity). Cache are used by flows's step to cache result to avoid recomputing - unnecessarily - - {/snippet} - - - {#snippet extra()} - - Themes are actually resources (but excluded from the Workspace tab for clarity). They - are CSS for the legacy low-code app editor only, and are added from the CSS panel of - an app rather than from this page. - - {/snippet} - - -
    -
    -
    - - {#if showTable} -
    - {#if loading.resources} - - {#each new Array(6) as _} - - {/each} - {:else if filteredItems?.length == 0} - {#if activeFilters} - - {:else} - appConnect?.open?.(), - aiId: 'resources-empty-add-resource', - aiDescription: 'Add resource' - } - : undefined} - /> - {/if} - {:else} - - - - - Path - Resource type - Description - Status - Actions - - - - {#if filteredItems} - {#each filteredItems as { path, description, resource_type, extra_perms, canWrite, is_oauth, is_linked, account, refresh_error, is_expired, marked, is_refreshed, labels, inherited_labels, ws_specific, draft_only, is_draft }} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'resource', path) ?? is_draft} - - - - - -
    - { - handledHash = `#/resource/${path}` - openResourceEditor(path, resource_type) - }} - >{#if marked}{@html marked}{:else}{path}{/if}{hasDraft ? '*' : ''} - - {#if labels?.length} -
    - {#each labels as label} - { - const arr = (filters.val.label ?? '').split(',').filter(Boolean) - const idx = arr.indexOf(label) - if (idx >= 0) arr.splice(idx, 1) - else arr.push(label) - const newFilters = { ...filters.val } - if (arr.length) newFilters.label = arr.join(',') - else delete newFilters.label - filters.val = newFilters - }}>{label} - {/each} -
    - {/if} - -
    -     - - { - const linkedRt = resourceTypes?.find((rt) => rt.name === resource_type) - if (linkedRt) { - resourceTypeViewerObj = { - rt: linkedRt.name, - //@ts-ignore - schema: linkedRt.schema, - description: linkedRt.description ?? '', - formatExtension: linkedRt.format_extension, - isFileset: linkedRt.is_fileset ?? false - } - resourceTypeViewer?.openDrawer?.() - } else { - sendUserToast( - `Resource type ${resource_type} not found in workspace.`, - true - ) - } - }} - > - - - - -
    - - {removeMarkdown(truncate(description ?? '', 200))} - -
    -     - -
    -
    - {#if is_linked} - - - {#snippet text()} -
    - This resource is linked with a variable of the same path. They - are deleted and renamed together. -
    - {/snippet} -     - {/if} -
    -
    - {#if is_refreshed} - - - {#snippet text()} -
    - The OAuth token will be kept up-to-date in the background by - Windmill using its refresh token -
    - {/snippet} -     - {/if} -
    - - {#if is_oauth} -
    - {#if refresh_error} - - - {#snippet text()} -
    - Latest exchange of the refresh token did not succeed. Error: {refresh_error} -
    - {/snippet} -     - {:else if is_expired} - - - - {#snippet text()} -
    - The access_token is expired, it will get renewed the next time - this variable is fetched or you can request is to be refreshed - in the dropdown on the right. -
    - {/snippet} -     - {:else} - - - {#snippet text()} -
    - The resource was connected through OAuth and the token is not - expired. -
    - {/snippet} -     - {/if} -
    - {/if} -
    -     - -
    - {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} - - {/if} - { - shareModal?.openDrawer?.(path, 'resource') - } - }, - { - displayName: 'Edit', - icon: Pen, - disabled: !canWrite || !showCreateButtons, - action: () => { - openResourceEditor(path, resource_type) - } - }, - // The agent form covers an agent's configuration, not everything a - // resource carries: the workspace-specific toggle in particular is - // only in the generic editor. JSON rather than that editor's form, - // which would render the configuration field by field and write a - // default into every one the agent leaves unset. Both write the same - // draft row, so this is a second view of the same edits. - ...(resource_type === 'ai_agent' - ? [ - { - displayName: 'Edit as JSON', - icon: Braces, - disabled: !canWrite || !showCreateButtons, - action: () => { - // The drawer anchors itself in the hash, which the deep-link - // effect would then read and route back to the agent editor. - // Claim it first, as the row's own link does. - handledHash = `#/resource/${path}` - resourceEditor?.initEdit?.(path, { json: true }) - } - } - ] - : []), - ...(!ws_specific && isDeployable('resource', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'resource') - } - } - ] - : []), - { - displayName: 'Delete', - disabled: !canWrite || !showCreateButtons, - icon: Trash, - type: 'delete', - action: (event) => { - // TODO - // @ts-ignore - if (event?.shiftKey) { - deleteResource(path, account) - } else { - deleteIsLinked = is_linked ?? false - deletePath = path - deleteConfirmedCallback = () => { - deleteResource(path, account) - } - } - } - }, - ...(account != undefined - ? [ - { - displayName: 'Refresh token', - icon: RotateCw, - action: async () => { - await OauthService.refreshToken({ - workspace: $workspaceStore ?? '', - id: account ?? 0, - requestBody: { - path - } - }) - sendUserToast('Token refreshed') - loadResources() - } - } - ] - : []) - ]} - /> -
    -     - {/each} - {/if} - -     - {/if} -
    - {:else if tab == 'types'} - {#if loading.types} - - {#each new Array(6) as _} - - {/each} - {:else if filteredResourceTypes?.length == 0} -
    - -
    - {:else} -
    - - - - Name - Description - Actions - - - - {#if filteredResourceTypes} - {#each filteredResourceTypes as { name, description, schema, canWrite, format_extension, is_fileset }} - - - { - resourceTypeViewerObj = { - rt: name, - //@ts-ignore - schema: schema, - description: description ?? '', - formatExtension: format_extension, - isFileset: is_fileset ?? false - } - - resourceTypeViewer?.openDrawer?.() - }} - > - - - - - -
    - - {removeMarkdown(truncate(description ?? '', 200))} - -
    -     - - {#if !canWrite} - -
    - - Shared globally - - This resource type is from the 'admins' workspace shared with all - workspaces - - -
    - {:else if $userStore?.is_admin || $userStore?.is_super_admin} -
    - - -
    - {:else} - - Non Editable - - Since resource types are shared with the whole workspace, only admins - can edit/delete them - - - {/if} -     -     - {/each} - {/if} - -     -
    - {/if} - {/if} -     -     -{/if} - - - - - - - { - loadResources() - }} -/> - - - -{#if agentEditorTarget()} - {#await import('$lib/components/flows/content/AgentEditorModal.svelte') then { default: AgentEditorModal }} - t.host === undefined} - onRenamed={(from, to) => { - void loadResources() - // Only while the dialog still shows the agent: closed mid-request, it already cleared the - // anchor, and writing it back would reopen the editor on refresh. - if (agentEditorTarget()?.path !== from) return - // Claimed first, as a row click does, so the deep-link effect does not reopen it. - handledHash = `#/resource/${to}` - setPageDrawerAnchor(RESOURCES_PATH, to) - }} - /> - {/await} -{/if} + diff --git a/frontend/src/routes/(root)/(logged)/routes/+page.svelte b/frontend/src/routes/(root)/(logged)/routes/+page.svelte index 53de5457d1..a79a1a0471 100644 --- a/frontend/src/routes/(root)/(logged)/routes/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/routes/+page.svelte @@ -1,592 +1,5 @@ - - - - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.triggers} - -{:else} - - -
    - - - -
    -     -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - routeEditor?.openNew(false), - aiId: 'routes-empty-add', - aiDescription: 'Add route' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { summary, workspace_id, workspaced_route, mode, path, edited_by, edited_at, script_path, route_path, is_flow, extra_perms, canWrite, marked, http_method, static_asset_config, retry, error_handler_path, error_handler_args, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_http', path) ?? is_draft} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - -
    -
    - - - routeEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {#if marked} - - {@html marked} - - {:else if summary} - {summary} - {:else} - {http_method.toUpperCase()} - /{isCloudHosted() || workspaced_route || globalHttpWorkspacedRoute - ? workspace_id + '/' + route_path - : route_path} - {/if}{hasDraft ? '*' : ''} -
    -
    - {path} -
    -
    - {#if static_asset_config} - file: {static_asset_config.s3} - {:else} - runnable: {script_path} - {/if} -
    -     - - - -
    - - onToggleMode(path, mode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'http', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - routeEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'routes' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'http_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - try { - await HttpTriggerService.deleteHttpTrigger({ - workspace: $workspaceStore ?? '', - path - }) - sendUserToast(`Successfully deleted HTTP route: ${path}`) - loadTriggers() - } catch (error) { - sendUserToast(error.body || error.message, true) - } - } - } - ]} - /> -
    -
    -
    -
    - {#if edited_by}
    edited by {edited_by}
    {/if} -
    {edited_by ? 'at ' : ''}{displayDate(edited_at)}
    -
    -
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     -{/if} - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte index e403c981de..88b21bc34c 100644 --- a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte @@ -1,683 +1,5 @@ - - - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.schedules} - -{:else} - - - - -
    -
    - - {#snippet children({ item })} - - - - {/snippet} - - -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !schedules?.length} - {#if activeFilters} - - {:else} - scheduleEditor?.openNew(false), - aiId: 'schedules-empty-add', - aiDescription: 'Add schedule' - }} - /> - {/if} - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, error, summary, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, extra_perms, canWrite, jobs, paused_until, labels, inherited_labels, draft_only, is_draft } (path)} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'trigger_schedule', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const avg_s = jobs - ? jobs.reduce((acc, x) => acc + x.duration_ms, 0) / jobs.length - : undefined} - -
    -
    - - - { - handledHash = `#${path}` - scheduleEditor?.openEdit(path, is_flow) - }} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {summary || script_path}{hasDraft ? '*' : ''} -
    -
    - schedule: {path} -
    -     - {#if labels?.length} - {#each labels as label} - { - const arr = (filters.val.label ?? '').split(',').filter(Boolean) - const idx = arr.indexOf(label) - if (idx >= 0) arr.splice(idx, 1) - else arr.push(label) - const newFilters = { ...filters.val } - if (arr.length) newFilters.label = arr.join(',') - else delete newFilters.label - filters.val = newFilters - }}>{label} - {/each} - {/if} - - - {#if paused_until && new Date(paused_until) > new Date()} -
    - Paused until {new Date(paused_until).toLocaleString()} -
    - {/if} - - - - - -
    - {#if error} - - - - - - {#snippet text()} -
    - The schedule disabled itself because there was an error scheduling the - next job: {error} -
    - {/snippet} -     - {/if} -
    - -
    - - {#key toggleResetVersions[path] ?? 0} - { - if (canWrite) { - setScheduleEnabled(path, e.detail) - } else { - sendUserToast('not enough permission', true) - // Permission denied — bump the row's reset - // counter so the Toggle remounts back to the - // prop value. Without this, the local - // `bind:checked` flip from the user's click - // stays stuck on. - bumpToggleReset(path) - } - }} - /> - {/key} -
    -
    - - - { - goto(href) - } - }, - { - displayName: `Duplicate schedule`, - icon: Copy, - action: () => { - scheduleEditor?.openNew(is_flow, script_path, path) - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - await ScheduleService.deleteSchedule({ - workspace: $workspaceStore ?? '', - path - }) - loadSchedules() - } - }, - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - scheduleEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'schedules' - } - }) - } - } - ] - : []), - { - displayName: 'View runs', - icon: List, - href: `${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true` - }, - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Run now', - icon: Play, - action: () => { - runScheduleNow(script_path, path, is_flow, $workspaceStore!) - } - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'schedule') - } - } - ]} - /> -
    -
    -
    - {#if loadingSchedulesWithJobStats} -
    - - Job stats loading... -
    - {:else} -
    - {#if avg_s} -
    Avg: {(avg_s / 1000).toFixed(2)}s
    - {/if} - {#each jobs ?? [] as job} - {@const h = (avg_s ? job.duration_ms / avg_s : 1) * 7 + 3} - - -
    -
    - -
    -     -     - {/each} -
    - {/if} -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} -
    - {nbDisplayed} items out of {items.length} - -
    - {/if} -     -{/if} - - { - loadSchedules() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 99fe4330cb..689afdbb9f 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -71,7 +71,7 @@ type PreviewTarget } from '$lib/components/sessions/previewRouter' import { toolReloadEffect, tabsToReload } from '$lib/components/sessions/previewReload' - import { pageItemForListPath } from '$lib/components/sessions/previewPaths' + import { isPageItemListPath, stripBase } from '$lib/components/sessions/previewPaths' import { leafKeyFor, loadKind, @@ -834,12 +834,11 @@ owner?.navigate({ type: 'item', item }) return } - // A row opened on a list page inside a preview tab: its editor opens as a tab of its - // own, leaving the list where it is. - if (d.type === 'wm.session.openPageItem') { - if (typeof d.pagePath !== 'string' || typeof d.path !== 'string') return - const ref = pageItemForListPath(d.pagePath, d.path) - if (ref) owner?.open({ type: 'pageitem', ref }) + // A preview frame navigating to a list page: re-point the active tab to the list + // mounted in process, whose rows open as tabs of their own. + if (d.type === 'wm.session.openList') { + if (typeof d.href !== 'string' || !isPageItemListPath(stripBase(d.href))) return + owner?.navigate({ type: 'page', href: d.href, label: previewLocationLabel(d.href) }) return } // A job clicked inside a preview tab: open the run detail in a NEW tab so the diff --git a/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte index 912d171d86..0f4155d6b3 100644 --- a/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sqs_triggers/+page.svelte @@ -1,567 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - SQS triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - sqsTriggerEditor?.openNew(false), - aiId: 'sqs-triggers-empty-add', - aiDescription: 'Add SQS trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, edited_by, error, edited_at, script_path, is_flow, extra_perms, canWrite, mode, server_id, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = getLocalDraftHint($workspaceStore, 'trigger_sqs', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = new Date()} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - sqsTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {path}{hasDraft ? '*' : ''} -
    -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - SQS trigger is starting... - {:else} - Could not connect to SQS{error ? ': ' + error : ''} - {/if} - {:else} - Disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    Connected to SQS{!server_id ? ' (shutting down...)' : ''}
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'sqs', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - sqsTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'sqs' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'sqs_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - try { - await SqsTriggerService.deleteSqsTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } catch (error) { - sendUserToast(error.body, true) - } - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 873e812987..839695361e 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -1,722 +1,5 @@ - - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.variables} - -{:else} - - - {#if showCreateButtons} -
    - {#if tab == 'contextual' && ($userStore?.is_admin || $userStore?.is_super_admin)} - - {:else} - - {/if} -
    - {/if} -     - (showCreateButtons = v)} /> - - - - { - loadVariables() - }} - /> - -
    - - - - {#snippet extra()} - - Contextual variables are passed as environment variables when running a script and - depends on the execution context. - - {/snippet} - - - {#if tab == 'workspace'} - - {/if} -
    - - {#if tab == 'workspace'} -
    - {#if !filteredItems} - - {#each new Array(3) as _} - - {/each} - {:else if filteredItems.length == 0} - {#if activeFilters} - - {:else} - variableEditor?.initNew(), - aiId: 'variables-empty-add-variable', - aiDescription: 'Add variable' - } - : undefined} - /> - {/if} - {:else} - - - - - Path - Value - Description - Status - Actions - - - - {#each filteredItems as { path, value, is_secret, description, extra_perms, canWrite, account, is_refreshed, is_expired, refresh_error, is_linked, labels, inherited_labels, ws_specific, draft_only, is_draft }} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'variable', path) ?? is_draft} - - - - - -
    - { - handledHash = `#${path}` - variableEditor?.editVariable(path) - }} - href="#{path}" - > - {path}{hasDraft ? '*' : ''} - - - {#if labels?.length} - {#each labels as label} - { - const arr = (filters.val.label ?? '').split(',').filter(Boolean) - const idx = arr.indexOf(label) - if (idx >= 0) arr.splice(idx, 1) - else arr.push(label) - const newFilters = { ...filters.val } - if (arr.length) newFilters.label = arr.join(',') - else delete newFilters.label - filters.val = newFilters - }}>{label} - {/each} - {/if} - -
    -     - - -
    - {#if value} - {truncate(value, 20)} - {:else} - ∗∗∗∗ - {/if} -
    - {#if is_secret} - - - {#snippet text()} - This item is secret - {/snippet} - - {/if} -     -     - - {truncate(description ?? '', 50)} - - - -
    - {#if is_linked} - - - {#snippet text()} -
    - This variable is linked with a resource of the same path. They are - deleted and renamed together. -
    - {/snippet} -     - {/if} - {#if account} - - - {#snippet text()} -
    - This OAuth token will be kept up-to-date in the background by - Windmill using its refresh token -
    - {/snippet} -     - {/if} - - {#if is_refreshed} -
    - {#if refresh_error} - - -
    - - -
    - - {#snippet text()} -
    - Latest exchange of the refresh token did not succeed. Error: {refresh_error} -
    - {/snippet} -     - {:else if is_expired} - - - {#snippet text()} -
    - The access_token is expired, it will get renewed the next time - this variable is fetched or you can request is to be refreshed - in the dropdown on the right. -
    - {/snippet} -     - {:else} - - - {#snippet text()} -
    - The variable was connected through OAuth and the token is not - expired. -
    - {/snippet} -     - {/if} -
    - {/if} -
    -     - - { - let owner = isOwner(path, $userStore, $workspaceStore) - return [ - { - displayName: 'Edit', - icon: Pen, - action: () => variableEditor?.editVariable(path), - disabled: !canWrite || !showCreateButtons - }, - { - displayName: 'Delete', - icon: Trash, - type: 'delete', - action: (event) => { - if (event['shiftKey']) { - deleteVariable(path, account) - } else { - deleteIsLinked = is_linked ?? false - deleteConfirmedCallback = () => { - deleteVariable(path, account) - } - } - }, - disabled: !owner || !showCreateButtons - }, - ...(!ws_specific && - isDeployable(is_secret ? 'secret' : 'variable', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'variable') - } - } - ] - : []), - { - displayName: 'Permissions', - action: () => { - shareModal?.openDrawer(path, 'variable') - }, - icon: Shield - }, - ...(account != undefined - ? [ - { - displayName: 'Refresh token', - icon: RefreshCw, - action: async () => { - await OauthService.refreshToken({ - workspace: $workspaceStore ?? '', - id: account ?? 0, - requestBody: { - path - } - }) - sendUserToast('Token refreshed') - loadVariables() - } - } - ] - : []) - ] - }} - /> - -     - {/each} - -     - {/if} -
    - {:else if tab == 'contextual'} -
    - {#if loading.contextual} - - {#each new Array(8) as _} - - {/each} - {:else} - - {#if contextualVariables.filter((x) => x.is_custom).length === 0} - contextualVariableEditor?.initNew(), - aiId: 'variables-empty-add-contextual-variable', - aiDescription: 'Add contextual variable' - } - : undefined} - /> - {:else} - x.is_custom)} - keys={['name', 'value']} - getRowActions={$userStore?.is_admin || $userStore?.is_super_admin - ? (row) => { - return [ - { - displayName: 'Edit', - action: () => contextualVariableEditor?.editVariable(row.name, row.value) - }, - { - displayName: 'Delete', - type: 'delete', - action: () => { - deleteContextualVariable(row) - } - } - ] - } - : undefined} - /> - {/if} - - !x.is_custom)} - keys={['name', 'value', 'description']} - /> - {/if} -
    - {/if} -     -     -{/if} - - { - deleteConfirmedCallback = undefined - }} - on:confirmed={() => { - if (deleteConfirmedCallback) { - deleteConfirmedCallback() - } - deleteConfirmedCallback = undefined - }} -> -
    - Are you sure you want to remove this variable? - {#if deleteIsLinked} - - This variable is linked with a resource of the same path. The linked resource will also be - deleted. - - {/if} - -
    - You can press - SHIFT - while removing a variable to bypass confirmation. -
    -     -
    -     + diff --git a/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte index 35a167c296..176c2ccbe5 100644 --- a/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte @@ -1,579 +1,5 @@ - - - - (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} -/> - - - - - - - {#if isCloudHosted()} - - WebSocket triggers are disabled in the multi-tenant cloud. - -
    - {/if} -
    -
    - -
    -
    Filter by path of
    - - {#snippet children({ item })} - - - {/snippet} - -
    - - -
    - {#if $userStore?.is_super_admin && $userStore.username.includes('@')} - - {:else if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} -
    -
    - {#if loading} - {#each new Array(6) as _} - - {/each} - {:else if !triggers?.length} - websocketTriggerEditor?.openNew(false), - aiId: 'websocket-triggers-empty-add', - aiDescription: 'Add WebSocket trigger' - }} - /> - {:else if items?.length} -
    - {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, url, is_flow, extra_perms, canWrite, marked, error, last_server_ping, server_id, mode, retry, error_handler_path, error_handler_args, labels, draft_only, is_draft } (path)} - {@const hasDraft = - getLocalDraftHint($workspaceStore, 'trigger_websocket', path) ?? is_draft} - {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} - {@const ping = last_server_ping ? new Date(last_server_ping) : undefined} - {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} - {@const effectiveMode = draft_only ? 'disabled' : mode} - {@const enabled = effectiveMode === 'enabled' || effectiveMode === 'suspended'} - -
    -
    - - - websocketTriggerEditor?.openEdit(path, is_flow)} - class="min-w-0 grow hover:underline decoration-gray-400" - > -
    - {#if marked} - - {@html marked} - - {:else} - {url.startsWith('$script:') - ? 'URL: ' + url.replace('$script:', 'result of script ') - : url.startsWith('$flow:') - ? 'URL: ' + url.replace('$flow:', 'result of flow ') - : url} - {/if}{hasDraft ? '*' : ''} -
    -
    - {path} -
    -
    - runnable: {script_path} -
    -     - - - -
    - {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} - - - - - - {#snippet text()} -
    - {#if enabled} - {#if !server_id} - WebSocket is starting... - {:else} - WebSocket is not connected{error ? ': ' + error : ''} - {/if} - {:else} - WebSocket was disabled because of an error: {error} - {/if} -
    - {/snippet} -     - {:else if enabled} - - - - - {#snippet text()} -
    - WebSocket is connected{!server_id ? ' (shutting down...)' : ''} -
    - {/snippet} -     - {/if} -
    - -
    - - onToggleMode(path, newMode)} - triggerMode={effectiveMode} - includeModalConfig={{ - triggerPath: path, - triggerKind: 'websocket', - runnableConfig: { - path: script_path, - kind: is_flow ? 'flow' : 'script', - retry, - errorHandlerPath: error_handler_path, - errorHandlerArgs: error_handler_args - } - }} - {canWrite} - hideToggleLabels - hideDropdown - /> -
    - -
    - - { - goto(href) - } - }, - ...(canWrite && !draft_only && mode !== 'suspended' - ? [ - { - displayName: 'Suspend job execution', - icon: Pause, - action: () => { - onToggleMode(path, 'suspended') - } - } - ] - : []), - { - displayName: canWrite ? 'Edit' : 'View', - icon: canWrite ? Pen : Eye, - action: () => { - websocketTriggerEditor?.openEdit(path, is_flow) - } - }, - ...(isDeployable('trigger', path, deployUiSettings) - ? [ - { - displayName: 'Deploy to prod/staging', - icon: FileUp, - action: () => { - deploymentDrawer?.openDrawer(path, 'trigger', { - triggers: { - kind: 'websockets' - } - }) - } - } - ] - : []), - { - displayName: 'Audit logs', - icon: Eye, - href: `${base}/audit_logs?resource=${path}` - }, - { - displayName: 'Permissions', - icon: Shield, - action: () => { - shareModal?.openDrawer(path, 'websocket_trigger') - } - }, - { - displayName: 'Delete', - type: 'delete', - icon: Trash, - disabled: !canWrite, - action: async () => { - await WebsocketTriggerService.deleteWebsocketTrigger({ - workspace: $workspaceStore ?? '', - path - }) - loadTriggers() - } - } - ]} - /> -
    -
    -
    -
    {#if edited_by}
    edited by {edited_by}
    {/if}
    {edited_by ? 'the ' : ''}{displayDate(edited_at)}
    -
    - {/each} -
    - {:else} - - {/if} -
    - {#if items && items?.length > 15 && nbDisplayed < items.length} - {nbDisplayed} items out of {items.length} - - {/if} -     - - { - loadTriggers() - }} -/> + From 2f1953ea10e8c62137a2b925ef2bd1eccf150736 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 24 Sep 2026 15:36:04 +0200 Subject: [PATCH 12/91] restore the operator alias in the offline seat-count query (#11336) * fix: restore the operator alias in the offline seat-count query Co-Authored-By: Claude Opus 5.5 (1M context) * chore: update ee-repo-ref to bf64ca98472a57cbfd150f8876877d2d4d9c217e This commit updates the EE repository reference after PR #827 was merged in windmill-ee-private. Previous ee-repo-ref: 59ac4f6e051fa9b6656b5d34759c09a04c4d8b85 New ee-repo-ref: bf64ca98472a57cbfd150f8876877d2d4d9c217e Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9d91911b1d..291341a9ae 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -48193da8cb30bc4ae82a50f944caef85d8a20b48 +bf64ca98472a57cbfd150f8876877d2d4d9c217e From 1fd729ac1aaec1e6a3af0d6db5fc1d6e1db6fa62 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 25 Sep 2026 11:23:30 +0200 Subject: [PATCH 13/91] feat: add an instance-wide accent color setting with sidebar tint (#11335) * feat: add an instance-wide accent color setting with sidebar tint Co-Authored-By: Claude Opus 5.5 (1M context) * fix: paint the cached accent before the license resolves Co-Authored-By: Claude Opus 5.5 (1M context) * feat: serve the banner and accent color from one cached endpoint Co-Authored-By: Claude Opus 5.5 (1M context) * fix: invalidate the instance ui cache and bound it with a ttl Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep the instance ui ttl under the client poll period Co-Authored-By: Claude Opus 5.5 (1M context) * refactor: load the banner and accent color once per page load Co-Authored-By: Claude Opus 5.5 (1M context) * refactor: read the banner and accent color without a server cache Co-Authored-By: Claude Opus 5.5 (1M context) * fix: show a cleared accent color as off Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/tests/instance_config.rs | 31 ++++ backend/windmill-api-settings/src/lib.rs | 24 ++- backend/windmill-api/openapi.yaml | 20 +++ .../windmill-common/src/global_settings.rs | 65 +++++++++ .../windmill-common/src/instance_config.rs | 8 + frontend/src/lib/accentColor.test.ts | 25 ++++ frontend/src/lib/accentColor.ts | 138 ++++++++++++++++++ .../src/lib/components/InstanceBanner.svelte | 52 +------ .../src/lib/components/InstanceSetting.svelte | 3 + .../src/lib/components/InstanceUiSync.svelte | 55 +++++++ frontend/src/lib/components/instanceBanner.ts | 6 - .../src/lib/components/instanceSettings.ts | 18 +++ .../AccentColorSetting.svelte | 51 +++++++ .../lib/components/sidebar/sidebarChrome.ts | 6 +- frontend/src/lib/instanceUi.ts | 9 ++ .../src/routes/(root)/(logged)/+layout.svelte | 11 +- frontend/tailwind.config.cjs | 11 +- 17 files changed, 467 insertions(+), 66 deletions(-) create mode 100644 frontend/src/lib/accentColor.test.ts create mode 100644 frontend/src/lib/accentColor.ts create mode 100644 frontend/src/lib/components/InstanceUiSync.svelte create mode 100644 frontend/src/lib/components/instanceSettings/AccentColorSetting.svelte create mode 100644 frontend/src/lib/instanceUi.ts diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index 5ab870a364..26a46d272e 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1608,3 +1608,34 @@ async fn declarative_sync_rejects_an_unusable_default_allowed_origins(db: Pool

    ) { + clear_settings_and_configs(&db).await; + + let mut desired = BTreeMap::new(); + desired.insert( + "accent_color".to_string(), + serde_json::json!("#000;}body{display:none"), + ); + let err = windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect_err("a non-hex accent color must fail the sync"); + assert!(err.to_string().contains("accent_color"), "got: {err}"); + assert!(get_global_setting(&db, "accent_color").await.is_none()); + + desired.insert("accent_color".to_string(), serde_json::json!("#1f9d55")); + windmill_common::instance_config::sync_global_settings_declarative( + &db, + &BTreeMap::new(), + &desired, + ) + .await + .expect("a hex accent color must sync"); +} diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index e8d8a7124c..bc52d85d54 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -58,9 +58,9 @@ use windmill_common::{ error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ - AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, + ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, + AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -115,6 +115,17 @@ async fn get_ruff_config_unauthed(Extension(db): Extension) -> error::Result .unwrap()) } +/// The announcement banner and accent color, which every signed-in session reads on each +/// full page load. +async fn get_instance_ui( + Extension(db): Extension, + _authed: ApiAuthed, +) -> JsonResult { + Ok(Json( + windmill_common::global_settings::get_instance_ui(&db).await?, + )) +} + pub fn global_service() -> Router { #[warn(unused_mut)] let r = Router::new() @@ -126,6 +137,7 @@ pub fn global_service() -> Router { "/global/{key}", post(set_global_setting).get(get_global_setting), ) + .route("/instance_ui", get(get_instance_ui)) .route("/list_global", get(list_global_settings)) .route("/github_app_stale_webhooks", get(github_app_stale_webhooks)) .route( @@ -1216,6 +1228,12 @@ async fn run_setting_pre_write_hook( } } } + ACCENT_COLOR_SETTING => match value { + serde_json::Value::Null => {} + serde_json::Value::String(s) if s.trim().is_empty() => {} + v => windmill_common::global_settings::validate_accent_color(v) + .map_err(|e| error::Error::BadRequest(format!("{ACCENT_COLOR_SETTING}: {e}")))?, + }, _ => {} } Ok(()) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0dd9f8620e..016342a6fe 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1762,6 +1762,26 @@ paths: schema: type: string + /settings/instance_ui: + get: + summary: get the instance-wide announcement banner and accent color + description: | + Readable by any authenticated user; every page load reads it. Each value + is the raw stored setting, or null when unset. + operationId: getInstanceUi + tags: + - setting + responses: + "200": + description: the instance_banner and accent_color settings + content: + application/json: + schema: + type: object + properties: + instance_banner: {} + accent_color: {} + /settings_u/ruff_config: get: summary: get instance ruff config (unauthenticated) diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 6b9aae519d..e72a2bb4e2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -281,6 +281,51 @@ pub fn validate_instance_banner(value: &serde_json::Value) -> Result<(), String> Ok(()) } +/// Instance-wide accent color (`#rrggbb`) that recolors the UI's accent tokens and +/// tints the sidebar, so each environment of a deployment is recognizable at a glance. +/// Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]. +pub const ACCENT_COLOR_SETTING: &str = "accent_color"; + +/// Validate an [`ACCENT_COLOR_SETTING`] value. +/// +/// Only `#rrggbb` is accepted: the value is interpolated into a stylesheet every user +/// loads, so anything looser is CSS injection into every session of the instance. +pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> { + let s = value + .as_str() + .ok_or_else(|| "must be a string".to_string())?; + let hex = s.strip_prefix('#').unwrap_or(""); + if hex.len() != 6 || !hex.chars().all(|c| c.is_ascii_hexdigit()) { + return Err("must be a hex color of the form #rrggbb".to_string()); + } + Ok(()) +} + +/// The settings every signed-in browser reads on each full page load. +#[derive(serde::Serialize, Clone, Debug, Default, PartialEq)] +pub struct InstanceUi { + pub instance_banner: Option, + pub accent_color: Option, +} + +pub async fn get_instance_ui(db: &Pool) -> error::Result { + let rows = sqlx::query!( + "SELECT name, value FROM global_settings WHERE name = ANY($1)", + &[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str] + ) + .fetch_all(db) + .await?; + let mut ui = InstanceUi::default(); + for row in rows { + match row.name.as_str() { + INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value), + ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value), + _ => {} + } + } + Ok(ui) +} + /// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. /// /// The receiver path is appended to it verbatim, so anything that doesn't @@ -939,6 +984,26 @@ mod tests { } } + #[test] + fn accent_color_accepts_only_hex_rgb() { + for ok in ["#1f9d55", "#ABCDEF"] { + assert!( + validate_accent_color(&serde_json::json!(ok)).is_ok(), + "{ok}" + ); + } + for bad in [ + serde_json::json!("1f9d55"), + serde_json::json!("#fff"), + serde_json::json!("red"), + serde_json::json!("#000000;} body{display:none"), + serde_json::json!("#12345g"), + serde_json::json!(123), + ] { + assert!(validate_accent_color(&bad).is_err(), "{bad}"); + } + } + #[test] fn instance_banner_rejects_unsafe_and_malformed_values() { // The link becomes the href of an anchor shown to every user of the instance, diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index f4884d736e..508379c3a4 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -1354,6 +1354,14 @@ pub async fn sync_global_settings_declarative( .map_err(|e| anyhow::anyhow!("{banner_key}: {e}"))?, } + let accent_key = crate::global_settings::ACCENT_COLOR_SETTING; + match desired.get(accent_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(color) => crate::global_settings::validate_accent_color(color) + .map_err(|e| anyhow::anyhow!("{accent_key}: {e}"))?, + } + // An origin list that cannot be parsed is dropped at boot, leaving the // empty default — which is no restriction at all. Rejecting it here is what // keeps a typo in a ConfigMap from silently widening CORS instance-wide. diff --git a/frontend/src/lib/accentColor.test.ts b/frontend/src/lib/accentColor.test.ts new file mode 100644 index 0000000000..6d3b855240 --- /dev/null +++ b/frontend/src/lib/accentColor.test.ts @@ -0,0 +1,25 @@ +import { describe, it, expect } from 'vitest' +import { accentStylesheet } from './accentColor' + +describe('accentStylesheet', () => { + it('refuses anything but #rrggbb, since the value lands in a stylesheet', () => { + for (const bad of ['#000;}body{display:none', '#fff', 'red', '1f9d55', 123, null]) { + expect(accentStylesheet(bad)).toBeUndefined() + } + }) + + it('overrides every accent token with an in-range r g b triplet in each theme', () => { + // A saturated color exercises the gamut clamp, a grey the zero-chroma path. + for (const color of ['#ff0000', '#808080']) { + const rules = accentStylesheet(color)!.split('\n') + expect(rules).toHaveLength(3) + for (const rule of rules) { + const triplets = [...rule.matchAll(/--color-[a-z-]+: (\d+) (\d+) (\d+);/g)] + expect(triplets).toHaveLength(10) + for (const [, ...channels] of triplets) { + for (const c of channels) expect(Number(c)).toBeLessThanOrEqual(255) + } + } + } + }) +}) diff --git a/frontend/src/lib/accentColor.ts b/frontend/src/lib/accentColor.ts new file mode 100644 index 0000000000..f11a96bb79 --- /dev/null +++ b/frontend/src/lib/accentColor.ts @@ -0,0 +1,138 @@ +import tokensFile from './assets/tokens/tokens.json' +import githubDarkTokens from './assets/tokens/githubDark.json' +import { darkModeName, lightModeName } from './assets/tokens/colorTokensConfig' + +/** Instance-wide accent color, `#rrggbb` (mirrors `ACCENT_COLOR_SETTING` in the backend). */ +export const ACCENT_COLOR_SETTING = 'accent_color' + +const ACCENT_TOKENS = [ + 'surface-accent-primary', + 'surface-accent-hover', + 'surface-accent-clicked', + 'surface-accent-selected', + 'surface-accent-secondary', + 'surface-accent-secondary-hover', + 'surface-accent-secondary-clicked', + 'text-accent', + 'border-accent', + 'border-selected' +] as const + +const STYLE_ID = 'wm-accent-color' +const ACCENT_ATTR = 'data-wm-accent' + +/** + * The stored setting as a color, or `undefined` for "use the default theme". + * + * The backend already rejects anything but `#rrggbb`, but the value is interpolated into a + * stylesheet, so this is the last place that can refuse a row written around the validator. + */ +export function parseAccentColor(value: unknown): string | undefined { + return typeof value === 'string' && /^#[0-9a-fA-F]{6}$/.test(value) + ? value.toLowerCase() + : undefined +} + +type Oklch = [l: number, c: number, h: number] + +function srgbToLinear(v: number): number { + return v <= 0.04045 ? v / 12.92 : Math.pow((v + 0.055) / 1.055, 2.4) +} + +function linearToSrgb(v: number): number { + return v <= 0.0031308 ? v * 12.92 : 1.055 * Math.pow(v, 1 / 2.4) - 0.055 +} + +function hexToOklch(hex: string): Oklch { + const [r, g, b] = [1, 3, 5].map((i) => srgbToLinear(parseInt(hex.slice(i, i + 2), 16) / 255)) + const l = Math.cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b) + const m = Math.cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b) + const s = Math.cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b) + const L = 0.2104542553 * l + 0.793617785 * m - 0.0040720468 * s + const A = 1.9779984951 * l - 2.428592205 * m + 0.4505937099 * s + const B = 0.0259040371 * l + 0.7827717662 * m - 0.808675766 * s + return [L, Math.hypot(A, B), Math.atan2(B, A)] +} + +/** Linear sRGB channels, possibly out of [0, 1] when the color is outside the gamut. */ +function oklchToLinearRgb([L, C, h]: Oklch): [number, number, number] { + const A = C * Math.cos(h) + const B = C * Math.sin(h) + const l = (L + 0.3963377774 * A + 0.2158037573 * B) ** 3 + const m = (L - 0.1055613458 * A - 0.0638541728 * B) ** 3 + const s = (L - 0.0894841775 * A - 1.291485548 * B) ** 3 + return [ + 4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s, + -1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s, + -0.0041960863 * l - 0.7034186147 * m + 1.707614701 * s + ] +} + +/** `r g b` triplet (the format the `--color-*` variables use), chroma reduced into gamut. */ +function oklchToRgbTriplet([L, C, h]: Oklch): string { + const inGamut = (c: number) => + oklchToLinearRgb([L, c, h]).every((v) => v >= -1e-4 && v <= 1 + 1e-4) + let lo = 0 + let hi = C + if (!inGamut(C)) { + for (let i = 0; i < 20; i++) { + const mid = (lo + hi) / 2 + if (inGamut(mid)) lo = mid + else hi = mid + } + } else lo = C + return oklchToLinearRgb([L, lo, h]) + .map((v) => Math.round(Math.min(1, Math.max(0, linearToSrgb(v))) * 255)) + .join(' ') +} + +// Below this chroma the chosen color reads as grey, and the accents fade with it. +const FULL_CHROMA = 0.1 + +/** + * Move each accent token to the chosen hue while keeping its own lightness, so every + * text/background pairing keeps the contrast the default palette was designed with. + * Taking the chosen color verbatim would put white text on a pale yellow button. + */ +function recolor(tokens: Record, accent: Oklch): string { + const chromaScale = Math.min(1, accent[1] / FULL_CHROMA) + return ACCENT_TOKENS.map((name) => { + const [l, c] = hexToOklch(tokens[name]) + return `--color-${name}: ${oklchToRgbTriplet([l, c * chromaScale, accent[2]])};` + }).join('') +} + +/** The accent overrides for every theme, or `undefined` when `value` is not `#rrggbb`. */ +export function accentStylesheet(value: unknown): string | undefined { + const color = parseAccentColor(value) + if (color == undefined) return undefined + const accent = hexToOklch(color) + // `:root[attr]` outranks the `html`, `html.dark` and `html.dark.github-dark` rules + // that define the default tokens, whatever order the stylesheets load in. + return [ + `:root[${ACCENT_ATTR}]{${recolor(tokensFile.tokens[lightModeName], accent)}` + + `--sidebar-bg-light:color-mix(in oklab, ${color} 18%, var(--sidebar-bg-light-base));}`, + `:root[${ACCENT_ATTR}].dark{${recolor(tokensFile.tokens[darkModeName], accent)}` + + `--sidebar-bg-dark:color-mix(in oklab, ${color} 22%, var(--sidebar-bg-dark-base));}`, + `:root[${ACCENT_ATTR}].dark.github-dark{${recolor(githubDarkTokens, accent)}}` + ].join('\n') +} + +/** Apply the instance accent to the whole document, or restore the default with `undefined`. */ +export function applyAccentColor(value: string | undefined): void { + const root = document.documentElement + let style = document.getElementById(STYLE_ID) + const css = accentStylesheet(value) + if (css == undefined) { + style?.remove() + root.removeAttribute(ACCENT_ATTR) + return + } + if (!style) { + style = document.createElement('style') + style.id = STYLE_ID + document.head.appendChild(style) + } + style.textContent = css + root.setAttribute(ACCENT_ATTR, '') +} diff --git a/frontend/src/lib/components/InstanceBanner.svelte b/frontend/src/lib/components/InstanceBanner.svelte index 9e47f9399a..0277cba4c1 100644 --- a/frontend/src/lib/components/InstanceBanner.svelte +++ b/frontend/src/lib/components/InstanceBanner.svelte @@ -3,60 +3,16 @@ import { classes as alertClasses, icons as alertIcons } from '$lib/components/common/alert/model' import { ExternalLink, X } from 'lucide-svelte' import { useLocalStorageValue } from '$lib/svelte5Utils.svelte' - import { - fetchInstanceBanner, - isInstanceBannerVisible, - type ResolvedInstanceBanner - } from './instanceBanner' - import { instanceSettingsSaved } from './instanceSettings' + import { isInstanceBannerVisible, resolveInstanceBanner } from './instanceBanner' + import { instanceUi } from '$lib/instanceUi' - // Mounted only on EE (see the render site in the logged layout), so the poll costs - // nothing on CE. An announcement is only worth broadcasting while it is current, hence - // polling rather than waiting for the next full page load: a session left open all day - // is exactly the one that needs to hear about the maintenance window. - const POLL_MS = 60_000 - - let banner = $state(undefined) + // Fetched by `InstanceUiSync`, which also carries the accent color. + let banner = $derived(resolveInstanceBanner($instanceUi?.instance_banner)) // Per-viewer, per-announcement. Holds the fingerprint of the dismissed banner, so // a new announcement shows up again for everyone who dismissed the previous one. const dismissed = useLocalStorageValue('instance_banner_dismissed', '', 'string') - // The poll, the tab-focus refresh and the post-save refresh all call `load()` and can - // overlap. Responses are not ordered, so without this a slow earlier fetch lands last - // and puts a retracted announcement back on screen until the next successful poll. - let latestLoad = 0 - - async function load() { - const generation = ++latestLoad - try { - const next = await fetchInstanceBanner() - if (generation === latestLoad) banner = next - } catch (e) { - // Keep whatever is on screen: a transient failure must not silently retract - // an announcement that is still in force. - console.warn('Could not fetch the instance banner', e) - } - } - - $effect(() => { - // Re-runs on save so a superadmin who just published an announcement sees it - // immediately rather than on the next poll or page load. - $instanceSettingsSaved - load() - const interval = setInterval(() => { - if (!document.hidden) load() - }, POLL_MS) - const onVisible = () => { - if (!document.hidden) load() - } - document.addEventListener('visibilitychange', onVisible) - return () => { - clearInterval(interval) - document.removeEventListener('visibilitychange', onVisible) - } - }) - let shown = $derived(isInstanceBannerVisible(banner, dismissed.val)) let palette = $derived(alertClasses[banner?.severity ?? 'info']) let Icon = $derived(alertIcons[banner?.severity ?? 'info']) diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index aedd9cdf6b..0da63ffbb0 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -27,6 +27,7 @@ import WebhookBaseUrlSetting from './instanceSettings/WebhookBaseUrlSetting.svelte' import WsConnectivityTest from './instanceSettings/WsConnectivityTest.svelte' import InstanceBannerSetting from './instanceSettings/InstanceBannerSetting.svelte' + import AccentColorSetting from './instanceSettings/AccentColorSetting.svelte' import IndexerMemorySettings from './instanceSettings/IndexerMemorySettings.svelte' import IndexerJobIndexSettings from './instanceSettings/IndexerJobIndexSettings.svelte' import IndexerLogIndexSettings from './instanceSettings/IndexerLogIndexSettings.svelte' @@ -872,6 +873,8 @@ {:else if setting.fieldType == 'instance_banner'} + {:else if setting.fieldType == 'accent_color'} + {/if} {#if hasError} diff --git a/frontend/src/lib/components/InstanceUiSync.svelte b/frontend/src/lib/components/InstanceUiSync.svelte new file mode 100644 index 0000000000..c9ae00eb29 --- /dev/null +++ b/frontend/src/lib/components/InstanceUiSync.svelte @@ -0,0 +1,55 @@ + diff --git a/frontend/src/lib/components/instanceBanner.ts b/frontend/src/lib/components/instanceBanner.ts index f1693024b8..4cceb7ad59 100644 --- a/frontend/src/lib/components/instanceBanner.ts +++ b/frontend/src/lib/components/instanceBanner.ts @@ -1,5 +1,3 @@ -import { SettingService } from '$lib/gen' - /** Drives the banner palette and icon; the subset of `AlertType` that fits an announcement. */ export type InstanceBannerSeverity = 'info' | 'warning' | 'error' @@ -123,7 +121,3 @@ export function isInstanceBannerVisible( if (banner == undefined) return false return !banner.dismissible || dismissedFingerprint !== banner.fingerprint } - -export async function fetchInstanceBanner(): Promise { - return resolveInstanceBanner(await SettingService.getGlobal({ key: INSTANCE_BANNER_SETTING })) -} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 1fe45931a2..92988f53b5 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -2,6 +2,7 @@ import type { ButtonType } from './common/button/model' import { allowedOriginsSettingError } from './triggers/http/utils' import { z } from 'zod' import { instanceBannerFormError } from './instanceBanner' +import { ACCENT_COLOR_SETTING, parseAccentColor } from '$lib/accentColor' import { parseMaxTokenExpirationDays } from '$lib/tokenExpiration' import { writable } from 'svelte/store' @@ -72,6 +73,7 @@ export interface Setting { | 'ws_connectivity' | 'retention_overrides' | 'instance_banner' + | 'accent_color' storage: SettingStorage advancedToggle?: { label: string @@ -334,6 +336,22 @@ export const settings: Record = { hideInQuickSetup: true, // Gates Save. The card renders the specific message itself, so no `error` here. isValid: (value: any) => instanceBannerFormError(value) == undefined + }, + { + label: 'Accent color', + description: + 'Recolors buttons, selections and the sidebar for every user, so each environment (e.g. staging vs. production) is recognizable at a glance.', + key: ACCENT_COLOR_SETTING, + fieldType: 'accent_color', + storage: 'setting', + ee_only: '', + hideInQuickSetup: true, + error: 'Must be a hex color like #1f9d55', + isValid: (value: unknown) => + value === undefined || + value === null || + value === '' || + parseAccentColor(value) !== undefined } ], Jobs: [ diff --git a/frontend/src/lib/components/instanceSettings/AccentColorSetting.svelte b/frontend/src/lib/components/instanceSettings/AccentColorSetting.svelte new file mode 100644 index 0000000000..619d32a7a0 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/AccentColorSetting.svelte @@ -0,0 +1,51 @@ + + +

    + enabled, (v) => ($values[ACCENT_COLOR_SETTING] = v ? DEFAULT_PICK : undefined) + } + options={{ right: 'Use a custom accent color' }} + /> + {#if enabled} +
    + ($values[ACCENT_COLOR_SETTING] = e.currentTarget.value)} + /> +
    + raw ?? '', (v) => ($values[ACCENT_COLOR_SETTING] = String(v).trim() || undefined)} + /> +
    +
    + + Accents keep the default palette's lightness and take this color's hue, so text stays readable + whatever color you pick. Applies to every user after saving. + + {/if} +
    diff --git a/frontend/src/lib/components/sidebar/sidebarChrome.ts b/frontend/src/lib/components/sidebar/sidebarChrome.ts index 71ff703d27..4c58202416 100644 --- a/frontend/src/lib/components/sidebar/sidebarChrome.ts +++ b/frontend/src/lib/components/sidebar/sidebarChrome.ts @@ -1,6 +1,6 @@ // Sidebar rail background. Kept here as the single source for every host that // renders or mimics the sidebar (app layout, kitchen-sink harness). -// The dark value is a theme-driven CSS variable so it adapts across dark -// variants (default vs GitHub dark) — see tailwind.config.cjs `--sidebar-bg-dark`. -export const SIDEBAR_BG = '#F3F3F7' +// Both are CSS variables (see tailwind.config.cjs `--sidebar-bg-*`) so they adapt +// across dark variants and to the instance accent tint (accentColor.ts). +export const SIDEBAR_BG = 'var(--sidebar-bg-light)' export const SIDEBAR_BG_DARK = 'var(--sidebar-bg-dark)' diff --git a/frontend/src/lib/instanceUi.ts b/frontend/src/lib/instanceUi.ts new file mode 100644 index 0000000000..b9b87d139c --- /dev/null +++ b/frontend/src/lib/instanceUi.ts @@ -0,0 +1,9 @@ +import { writable } from 'svelte/store' +import type { GetInstanceUiResponse } from '$lib/gen' + +/** + * The last `/settings/instance_ui` response: the announcement banner and accent color. + * `undefined` until one lands, and left as is when a fetch fails, so a transient error + * never retracts an announcement still in force. Written only by `InstanceUiSync`. + */ +export const instanceUi = writable(undefined) diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index e9d4493e66..25d2fd156f 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -81,6 +81,7 @@ import { pruneMeaninglessDrafts } from '$lib/userDraftPrune' import DraftMigrationErrorModal from '$lib/components/DraftMigrationErrorModal.svelte' import InstanceBanner from '$lib/components/InstanceBanner.svelte' + import InstanceUiSync from '$lib/components/InstanceUiSync.svelte' import { onDestroy, setContext, untrack } from 'svelte' import { base } from '$app/paths' import { Menubar } from '$lib/components/meltComponents' @@ -1437,11 +1438,13 @@
    {/if}
    + + {#if $enterpriseLicense && !menuHidden} - + {/if} {#if $userStore?.is_service_account} diff --git a/frontend/tailwind.config.cjs b/frontend/tailwind.config.cjs index a5aea63715..5c623fbadc 100644 --- a/frontend/tailwind.config.cjs +++ b/frontend/tailwind.config.cjs @@ -603,6 +603,12 @@ const config = { backgroundColor: 'rgb(var(--color-surface-primary))', color: 'rgb(var(--color-text-primary))', + // Sidebar rail chrome — consumed via var(--sidebar-bg-light) (see sidebarChrome.ts). + // The `-base` vars hold the theme's color; the accent (accentColor.ts) tints + // `--sidebar-bg-*` from them, which a self-reference could not do. + '--sidebar-bg-light-base': '#F3F3F7', + '--sidebar-bg-light': 'var(--sidebar-bg-light-base)', + '--vscode-editorSuggestWidget-background': '#f3f3f3', '--vscode-editorHoverWidget-foreground': '#616161', '--vscode-editorHoverWidget-border': '#c8c8c8', @@ -622,7 +628,8 @@ const config = { ...darkColorVariables, // Sidebar rail chrome — consumed via var(--sidebar-bg-dark) (see sidebarChrome.ts). - '--sidebar-bg-dark': '#1e232e', + '--sidebar-bg-dark-base': '#1e232e', + '--sidebar-bg-dark': 'var(--sidebar-bg-dark-base)', '--vscode-editorSuggestWidget-background': '#252526', '--vscode-editorHoverWidget-foreground': '#cccccc', @@ -638,7 +645,7 @@ const config = { // GitHub's canvas.inset — the authentic recessed surface, sitting below // the canvas (#0d1117) like the sidebar-vs-primary relationship in the // default dark theme. - '--sidebar-bg-dark': '#010409', + '--sidebar-bg-dark-base': '#010409', // Monaco popup chrome — the &.dark defaults above are VS Code's greys; // override so the widgets match the GitHub Dark palette in this variant. From 0bac766756cf9ebcb615fe20d5a69c09c823c725 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 25 Sep 2026 11:38:41 +0200 Subject: [PATCH 14/91] feat: refresh MCP tools when scripts and flows change (#11337) * feat: notify MCP clients when a workspace's scripts or flows change Advertise tools.listChanged and implement subscriptions/listen, so a 2026-07-28 client refreshes its tool list when scripts or flows are deployed, archived, renamed or deleted. Changes reach every replica via new statement-level notify_event triggers; MCP-originated changes also signal the serving replica inline. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: signal each workspace once per notify-event poll batch Co-Authored-By: Claude Opus 5.5 (1M context) * fix: signal script/flow path moves, keep unrelated updates trigger-free Row-level UPDATE OF archived/deleted/path triggers with a WHEN guard replace the statement-level ones, which built transition tables for every UPDATE on script and flow. Path moves from username changes and offboarding are now signalled. subscriptions/listen is refused when the client asks for nothing this server sends. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: signal path moves only for unarchived versions A username change or offboarding rewrites the path of every version, archived ones included, which would queue one notify_event per version. Co-Authored-By: Claude Opus 5.5 (1M context) * refactor: detect MCP tool-list changes by polling a workspace fingerprint Replace the notify_event triggers and poller hook with a per-process, per-workspace poll of a hash over the live scripts and flows, run only while a subscriptions/listen stream watches that workspace. Every write path (UI, CLI, git sync, user renames, workspace moves) is covered with no migration; an MCP-originated change polls at once. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: announce the first fingerprint so a change before the baseline is not lost Also poll immediately after a script/flow mutation through a multi-workspace token, and pin the fingerprint test on a lock update, which is the unrelated write that actually happens. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: open every tool-list subscription with one notification A subscriber joining an existing watcher missed a change the watcher recorded before it subscribed. Also move the fingerprint query to a runnables module, since it spans scripts and flows. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/windmill-api/src/mcp/core.rs | 6 + backend/windmill-common/src/lib.rs | 1 + backend/windmill-common/src/runnables.rs | 22 +++ .../tests/runnable_list_fingerprint.rs | 54 +++++++ backend/windmill-mcp/src/server/backend.rs | 5 + .../windmill-mcp/src/server/list_changes.rs | 104 +++++++++++++ backend/windmill-mcp/src/server/mod.rs | 1 + backend/windmill-mcp/src/server/runner.rs | 142 ++++++++++++++++-- 8 files changed, 321 insertions(+), 14 deletions(-) create mode 100644 backend/windmill-common/src/runnables.rs create mode 100644 backend/windmill-common/tests/runnable_list_fingerprint.rs create mode 100644 backend/windmill-mcp/src/server/list_changes.rs diff --git a/backend/windmill-api/src/mcp/core.rs b/backend/windmill-api/src/mcp/core.rs index 86fbec1806..472c7bbec9 100644 --- a/backend/windmill-api/src/mcp/core.rs +++ b/backend/windmill-api/src/mcp/core.rs @@ -386,6 +386,12 @@ impl McpBackend for WindmillBackend { }) } + async fn runnable_list_fingerprint(&self, workspace_id: &str) -> BackendResult { + windmill_common::runnables::runnable_list_fingerprint(&self.db, workspace_id) + .await + .map_err(|e| ErrorData::internal_error(e.to_string(), None)) + } + fn all_endpoint_tools(&self) -> Vec { all_tools() } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 3e74d8e17f..1015948882 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -118,6 +118,7 @@ pub mod queue; pub mod queue_metrics; pub mod result_stream; pub mod runnable_settings; +pub mod runnables; pub mod schedule; pub mod schema; pub mod scripts; diff --git a/backend/windmill-common/src/runnables.rs b/backend/windmill-common/src/runnables.rs new file mode 100644 index 0000000000..579e5e248e --- /dev/null +++ b/backend/windmill-common/src/runnables.rs @@ -0,0 +1,22 @@ +//! Queries spanning both scripts and flows. + +use crate::{error::Error, DB}; + +/// An opaque value that changes whenever a workspace's live scripts or flows are added, +/// removed, moved or redeployed: a redeploy is a new script hash or a new last flow +/// version, so schema and description changes register too. Not filtered by caller, +/// so it reveals nothing beyond whether it changed. +pub async fn runnable_list_fingerprint(db: &DB, workspace_id: &str) -> Result { + Ok(sqlx::query_scalar::<_, String>( + "SELECT COALESCE(md5(string_agg(k, ',' ORDER BY k)), '') FROM ( + SELECT 's' || path || ':' || hash FROM script + WHERE workspace_id = $1 AND archived = false + UNION ALL + SELECT 'f' || path || ':' || COALESCE(versions[array_upper(versions, 1)], 0) FROM flow + WHERE workspace_id = $1 AND archived = false + ) t(k)", + ) + .bind(workspace_id) + .fetch_one(db) + .await?) +} diff --git a/backend/windmill-common/tests/runnable_list_fingerprint.rs b/backend/windmill-common/tests/runnable_list_fingerprint.rs new file mode 100644 index 0000000000..d7438f0008 --- /dev/null +++ b/backend/windmill-common/tests/runnable_list_fingerprint.rs @@ -0,0 +1,54 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::runnables::runnable_list_fingerprint; + +/// MCP clients refresh their tool list when this fingerprint moves, so it must move on every +/// change to what the listing shows and stay put on writes it does not show. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn fingerprint_tracks_the_live_runnable_list(db: Pool) { + let fp = || async { + runnable_list_fingerprint(&db, "test-workspace") + .await + .unwrap() + }; + let exec = |sql: &'static str| { + let db = db.clone(); + async move { sqlx::query(sql).execute(&db).await.unwrap() } + }; + + let mut last = fp().await; + let mut assert_moved = async |what: &str, moved: bool| { + let now = fp().await; + assert_eq!(now != last, moved, "{what}"); + last = now; + }; + + exec( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, language, kind) + VALUES ('test-workspace', 1, 'f/test/a', '', '', 'def main(): pass', 'test-user', 'python3', 'script')", + ) + .await; + assert_moved("deploy", true).await; + + exec("UPDATE script SET lock = '' WHERE hash = 1").await; + assert_moved("lock update", false).await; + + exec("UPDATE script SET path = 'f/test/b' WHERE hash = 1").await; + assert_moved("path move", true).await; + + exec("UPDATE script SET archived = true WHERE hash = 1").await; + assert_moved("archive", true).await; + + exec( + "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, schema, versions) + VALUES ('test-workspace', 'f/test/flow', '', '', '{}'::jsonb, 'test-user', '{}'::json, ARRAY[1]::bigint[])", + ) + .await; + assert_moved("flow create", true).await; + + exec("UPDATE flow SET versions = array_append(versions, 2::bigint) WHERE path = 'f/test/flow'") + .await; + assert_moved("flow redeploy", true).await; + + exec("DELETE FROM flow WHERE path = 'f/test/flow'").await; + assert_moved("flow delete", true).await; +} diff --git a/backend/windmill-mcp/src/server/backend.rs b/backend/windmill-mcp/src/server/backend.rs index 7ca1eb08a1..4801bfaa99 100644 --- a/backend/windmill-mcp/src/server/backend.rs +++ b/backend/windmill-mcp/src/server/backend.rs @@ -208,6 +208,11 @@ pub trait McpBackend: Send + Sync + Clone + 'static { workspace_id: &str, ) -> BackendResult; + /// An opaque value that changes whenever the workspace's live scripts or flows are + /// added, removed, moved or redeployed, compared across polls to detect tool-list + /// changes. + async fn runnable_list_fingerprint(&self, workspace_id: &str) -> BackendResult; + // ───────────────────────────────────────────────────────────────── // Endpoint Tools // ───────────────────────────────────────────────────────────────── diff --git a/backend/windmill-mcp/src/server/list_changes.rs b/backend/windmill-mcp/src/server/list_changes.rs new file mode 100644 index 0000000000..8470cff630 --- /dev/null +++ b/backend/windmill-mcp/src/server/list_changes.rs @@ -0,0 +1,104 @@ +//! Detects changes to a workspace's scripts and flows for `subscriptions/listen`. +//! +//! Each process polls a fingerprint of every workspace one of its listeners watches, +//! so a change made on any replica, through any path (UI, CLI, git sync, a user +//! rename), reaches every listener within one poll interval with no cross-replica +//! signalling. + +use crate::server::backend::McpBackend; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::sync::{broadcast, Notify}; + +const POLL_INTERVAL: Duration = Duration::from_secs(30); + +struct Watcher { + changes: broadcast::Sender<()>, + poll_now: Arc, +} + +#[derive(Default)] +pub(crate) struct ListChanges { + watchers: Mutex>, +} + +impl ListChanges { + /// Subscribe to `workspace_id`'s changes, starting its poller if this is the + /// first listener for it in this process. + pub(crate) fn subscribe( + self: &Arc, + backend: &Arc, + workspace_id: &str, + ) -> broadcast::Receiver<()> { + let mut watchers = self.watchers.lock().unwrap(); + if let Some(watcher) = watchers.get(workspace_id) { + return watcher.changes.subscribe(); + } + let (changes, rx) = broadcast::channel(16); + let poll_now = Arc::new(Notify::new()); + watchers.insert( + workspace_id.to_string(), + Watcher { changes: changes.clone(), poll_now: poll_now.clone() }, + ); + tokio::spawn(self.clone().poll( + backend.clone(), + workspace_id.to_string(), + changes, + poll_now, + )); + rx + } + + /// Poll `workspace_id` now rather than at its next interval, after this process + /// changed it: the client that made the change is the one waiting on it. Polling + /// rather than signalling directly keeps the poller's baseline current, so the + /// same change is not reported again on the next tick. + pub(crate) fn poll_now(&self, workspace_id: &str) { + if let Some(watcher) = self.watchers.lock().unwrap().get(workspace_id) { + watcher.poll_now.notify_one(); + } + } + + async fn poll( + self: Arc, + backend: Arc, + workspace_id: String, + changes: broadcast::Sender<()>, + poll_now: Arc, + ) { + let mut interval = tokio::time::interval(POLL_INTERVAL); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut last = None; + loop { + tokio::select! { + _ = interval.tick() => {} + _ = poll_now.notified() => {} + } + { + // Checked under the lock `subscribe` takes, so a listener cannot join a + // watcher that is about to stop. + let mut watchers = self.watchers.lock().unwrap(); + if changes.receiver_count() == 0 { + watchers.remove(&workspace_id); + return; + } + } + match backend.runnable_list_fingerprint(&workspace_id).await { + Ok(fingerprint) => { + // The first fingerprint also notifies: the listener that started this + // watcher has already sent its opening notification, and a change landing + // before this baseline would otherwise never be announced. + if last.as_ref() != Some(&fingerprint) { + let _ = changes.send(()); + } + last = Some(fingerprint); + } + Err(e) => tracing::warn!( + "MCP tool-list poll failed for workspace {workspace_id}: {}", + e.message + ), + } + } + } +} diff --git a/backend/windmill-mcp/src/server/mod.rs b/backend/windmill-mcp/src/server/mod.rs index b97e374e98..ad7c14d320 100644 --- a/backend/windmill-mcp/src/server/mod.rs +++ b/backend/windmill-mcp/src/server/mod.rs @@ -7,6 +7,7 @@ pub mod backend; pub mod endpoints; +mod list_changes; pub mod runner; pub mod tools; diff --git a/backend/windmill-mcp/src/server/runner.rs b/backend/windmill-mcp/src/server/runner.rs index e27eaa8470..177989a82e 100644 --- a/backend/windmill-mcp/src/server/runner.rs +++ b/backend/windmill-mcp/src/server/runner.rs @@ -16,20 +16,22 @@ use crate::server::backend::{McpAuth, McpBackend, McpRequest, PathFilter}; use crate::server::endpoints::{ endpoint_tool_to_mcp_tool, endpoint_tool_to_mcp_tool_multi, list_workspaces_tool, EndpointTool, }; +use crate::server::list_changes::ListChanges; use crate::server::tools::create_tool_from_item; use rmcp::handler::server::ServerHandler; use rmcp::model::{ CacheScope, CallToolRequestParams, CallToolResponse, CallToolResult, ContentBlock, - Implementation, InitializeResult, ListPromptsResult, ListResourceTemplatesResult, - ListResourcesResult, ListToolsResult, PaginatedRequestParams, ProtocolVersion, - ServerCapabilities, ServerInfo, + Implementation, InitializeRequestParams, InitializeResult, ListPromptsResult, + ListResourceTemplatesResult, ListResourcesResult, ListToolsResult, PaginatedRequestParams, + ProtocolVersion, ServerCapabilities, ServerInfo, SubscriptionFilter, }; -use rmcp::service::{RequestContext, RoleServer}; +use rmcp::service::{RequestContext, RoleServer, SubscriptionContext}; use rmcp::ErrorData; use serde_json::Value; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use tokio::sync::broadcast; /// Protocol revisions this server is willing to speak. `2026-07-28` is served /// statelessly with per-request metadata; the older revisions keep the @@ -46,15 +48,21 @@ const SUPPORTED_PROTOCOL_VERSIONS: &[ProtocolVersion] = &[ /// leaves them unset, and a strict client rejects the response without them. /// /// Zero because nothing here is cacheable: the listing is rebuilt from the -/// workspace's scripts and flows, which change at any time, and this server -/// advertises no `listChanged` capability, so a client that cached a stale list -/// would have no way to learn it had gone stale. +/// workspace's scripts and flows, which change at any time, and `listChanged` +/// covers only some of what the listing depends on (not resources, favorites or +/// permissions), lags by up to a poll interval, and reaches only clients that hold a +/// `subscriptions/listen` stream. const LIST_TTL_MS: u64 = 0; + /// Every listing is filtered by the caller's token scopes and workspace /// membership, so no two callers necessarily see the same tools — a shared /// cache entry would leak one token's view to another. const LIST_CACHE_SCOPE: CacheScope = CacheScope::Private; +/// How long a `subscriptions/listen` stream waits after a change before notifying, +/// collecting the rest of the burst into the same notification. +const LIST_CHANGE_COALESCE: std::time::Duration = std::time::Duration::from_millis(500); + // Re-export from http crate for extracting request parts use http::request::Parts as HttpParts; @@ -83,11 +91,12 @@ fn truncate_tool_result(text: String) -> String { /// to perform the actual operations (database queries, job execution, etc.) pub struct Runner { backend: Arc, + list_changes: Arc, } impl Clone for Runner { fn clone(&self) -> Self { - Self { backend: self.backend.clone() } + Self { backend: self.backend.clone(), list_changes: self.list_changes.clone() } } } @@ -113,7 +122,7 @@ struct McpContext { impl Runner { /// Create a new Runner with the given backend pub fn new(backend: B) -> Self { - Self { backend: Arc::new(backend) } + Self { backend: Arc::new(backend), list_changes: Arc::default() } } /// Extract authentication, the workspace mode and the HTTP request itself @@ -232,6 +241,22 @@ fn endpoint_path_policy(endpoint_name: &str) -> Option { } } +/// Endpoint tools that change which scripts/flows a workspace exposes, or their +/// schemas. The process serving the call polls the workspace right away rather than +/// at its next interval, so a client that just deployed a script can call it. +fn changes_runnable_list(endpoint_name: &str) -> bool { + matches!( + endpoint_name, + "createScript" + | "updateScript" + | "deleteScriptByPath" + | "deleteScriptByHash" + | "createFlow" + | "updateFlow" + | "deleteFlowByPath" + ) +} + /// Whether the token restricts `kind` ("script"/"flow") to specific paths. A /// `*` pattern grants every path (see `is_resource_allowed`), so it does not /// count as confinement. @@ -428,12 +453,17 @@ impl ServerHandler for Runner { .with_title("Windmill") .with_website_url("https://windmill.dev"); - InitializeResult::new(ServerCapabilities::builder().enable_tools().build()) - .with_server_info(server_info) - .with_instructions( - "This server provides a list of scripts and flows the user can run on Windmill. \ + InitializeResult::new( + ServerCapabilities::builder() + .enable_tools() + .enable_tool_list_changed() + .build(), + ) + .with_server_info(server_info) + .with_instructions( + "This server provides a list of scripts and flows the user can run on Windmill. \ Each flow and script is a tool callable with their respective arguments.", - ) + ) } /// Pinned rather than left to rmcp's default (every version the SDK knows), so a @@ -444,6 +474,84 @@ impl ServerHandler for Runner { Cow::Borrowed(SUPPORTED_PROTOCOL_VERSIONS) } + /// Only clients below `2026-07-28` send `initialize`, and those receive server + /// notifications on a session's standalone stream. The transport is sessionless, so + /// there is none: `listChanged` is withheld from them rather than promised and + /// never delivered. `2026-07-28` clients learn it from `server/discover` and + /// receive it on `subscriptions/listen` (see `listen`). + async fn initialize( + &self, + request: InitializeRequestParams, + context: RequestContext, + ) -> Result { + context.peer.set_peer_info(request.clone()); + let mut info = self.get_info(); + if let Some(tools) = info.capabilities.tools.as_mut() { + tools.list_changed = None; + } + if SUPPORTED_PROTOCOL_VERSIONS.contains(&request.protocol_version) { + info.protocol_version = request.protocol_version; + } + Ok(info) + } + + /// `None` for a request that asks for nothing this server sends, so the client is + /// told the subscription is unsupported rather than acked with an empty filter. + fn accepted_subscription_filter( + &self, + requested: &SubscriptionFilter, + ) -> Option { + (requested.tools_list_changed == Some(true)) + .then(|| SubscriptionFilter::builder().tools_list_changed().build()) + } + + async fn listen(&self, subscription: SubscriptionContext) -> Result<(), ErrorData> { + let McpContext { auth, mode, .. } = Self::extract_context(subscription.request_context())?; + // Multi-workspace mode and read-only tokens list no scripts or flows, so their + // tool list never changes. + let workspace_id = match mode { + McpMode::Single(workspace_id) if !auth.read_only() => workspace_id, + _ => { + subscription.cancelled().await; + return Ok(()); + } + }; + + let mut changes = self.list_changes.subscribe(&self.backend, &workspace_id); + // Every stream opens with one notification: a change the watcher recorded between + // this client's `tools/list` and this subscription is never broadcast again. + let mut opening = true; + loop { + if !std::mem::take(&mut opening) { + tokio::select! { + _ = subscription.cancelled() => return Ok(()), + change = changes.recv() => match change { + Ok(()) | Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => return Ok(()), + }, + } + } + // A client creating several scripts in a row: collapse the burst into one + // notification, so it re-lists once. + tokio::select! { + _ = subscription.cancelled() => return Ok(()), + _ = tokio::time::sleep(LIST_CHANGE_COALESCE) => {} + } + while !matches!( + changes.try_recv(), + Err(broadcast::error::TryRecvError::Empty | broadcast::error::TryRecvError::Closed) + ) {} + if subscription + .sink() + .notify_tool_list_changed() + .await + .is_err() + { + return Ok(()); + } + } + } + async fn list_tools( &self, _request: Option, @@ -714,6 +822,9 @@ impl Runner { .backend .call_endpoint(auth, workspace_id, endpoint_tool, args) .await?; + if changes_runnable_list(&endpoint_tool.name) { + self.list_changes.poll_now(workspace_id); + } return Ok(CallToolResult::success(vec![ContentBlock::text( truncate_tool_result( @@ -969,6 +1080,9 @@ impl Runner { .backend .call_endpoint(&resolved_auth, &workspace_id, endpoint_tool, args) .await?; + if changes_runnable_list(&endpoint_tool.name) { + self.list_changes.poll_now(&workspace_id); + } Ok(CallToolResult::success(vec![ContentBlock::text( truncate_tool_result( From d18d7043df6536e10b22ee9f3df77e926a0460b4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 25 Sep 2026 12:00:49 +0200 Subject: [PATCH 15/91] feat: infer a script's schema when a deploy (e.g. MCP) sends none (#11339) * feat: infer a script's schema from its code when a deploy sends none Co-Authored-By: Claude Opus 5.5 (1M context) * fix: merge an inferred schema into the previous one the way the editor does Co-Authored-By: Claude Opus 5.5 (1M context) * fix: parse non-JSON TS defaults natively and keep the schema on a failed inference Co-Authored-By: Claude Opus 5.5 (1M context) * fix: type untyped TS params from their literal shape when the default can't be evaluated Co-Authored-By: Claude Opus 5.5 (1M context) * fix: read literal TS defaults off the AST so the server types them like the editor Co-Authored-By: Claude Opus 5.5 (1M context) * docs: word the script schema description for both create and update Co-Authored-By: Claude Opus 5.5 (1M context) * docs: note that dbt scripts derive their schema from the descriptor Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/Cargo.lock | 4 + .../generate_mcp_tools.py | 9 +- backend/parsers/windmill-parser-ts/src/lib.rs | 91 +++- .../parsers/windmill-parser-ts/tests/tests.rs | 32 ++ .../windmill-parser/src/json_schema.rs | 402 ++++++++++++++++++ backend/parsers/windmill-parser/src/lib.rs | 1 + backend/tests/script_schema_inference.rs | 115 +++++ backend/windmill-api-scripts/Cargo.toml | 4 + backend/windmill-api-scripts/src/scripts.rs | 124 ++++++ backend/windmill-api/openapi.yaml | 15 + .../src/mcp/auto_generated_endpoints.rs | 20 +- frontend/src/lib/mcpEndpointTools.ts | 20 +- 12 files changed, 817 insertions(+), 20 deletions(-) create mode 100644 backend/parsers/windmill-parser/src/json_schema.rs create mode 100644 backend/tests/script_schema_inference.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 57b15bc5a7..87bf7f64be 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15411,6 +15411,7 @@ dependencies = [ name = "windmill-api-scripts" version = "1.817.0" dependencies = [ + "anyhow", "axum 0.8.9", "chrono", "futures", @@ -15435,6 +15436,9 @@ dependencies = [ "windmill-native-triggers", "windmill-object-store", "windmill-parser", + "windmill-parser-bash", + "windmill-parser-go", + "windmill-parser-graphql", "windmill-parser-py", "windmill-parser-py-asset", "windmill-parser-sql", diff --git a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py index 72feb5392e..4c78f0524e 100644 --- a/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py +++ b/backend/generate_mcp_endpoints_tools/generate_mcp_tools.py @@ -128,11 +128,16 @@ def extract_separate_schemas(parameters: List[Dict[str, Any]], request_body: Opt r for r in body_schema['required'] if r in include_fields ] - # Apply opaque_fields: simplify listed properties to {"type": "object"} + # Apply opaque_fields: simplify listed properties to {"type": "object"}, + # keeping the description, which is all a caller learns about the field if body_schema and opaque_fields and 'properties' in body_schema: for field in opaque_fields: if field in body_schema['properties']: - body_schema['properties'][field] = {"type": "object"} + opaque = {"type": "object"} + description = body_schema['properties'][field].get('description') + if description: + opaque['description'] = description + body_schema['properties'][field] = opaque # If we have required fields specified and a body schema, update the required array if body_schema and required_fields: diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 271ad4835b..ac75080994 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -16,11 +16,13 @@ use windmill_parser::{ use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Span, Spanned}; use swc_ecma_ast::{ - ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, Ident, IdentName, Lit, - MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, ObjectLit, ObjectPat, Param, Pat, Stmt, - Str, TsArrayType, TsEntityName, TsInterfaceDecl, TsKeywordType, TsKeywordTypeKind, TsLit, - TsLitType, TsOptionalType, TsParenthesizedType, TsPropertySignature, TsType, TsTypeAliasDecl, - TsTypeAnn, TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType, + ArrayLit, AssignPat, BigInt, BindingIdent, Bool, Decl, ExportDecl, Expr, ExprOrSpread, Ident, + IdentName, KeyValueProp, Lit, MemberExpr, MemberProp, ModuleDecl, ModuleItem, Number, + ObjectLit, ObjectPat, Param, Pat, Prop, PropName, PropOrSpread, Stmt, Str, TsArrayType, + TsEntityName, TsInterfaceDecl, TsKeywordType, TsKeywordTypeKind, TsLit, TsLitType, + TsOptionalType, TsParenthesizedType, TsPropertySignature, TsType, TsTypeAliasDecl, TsTypeAnn, + TsTypeElement, TsTypeLit, TsTypeRef, TsUnionOrIntersectionType, TsUnionType, UnaryExpr, + UnaryOp, }; use swc_ecma_parser::{lexer::Lexer, EsSyntax, Parser, StringInput, Syntax, TsSyntax}; @@ -578,6 +580,15 @@ fn parse_param( } }; + let literal = match &*right { + Expr::Object(_) | Expr::Array(_) if !skip_dflt => literal_json(&right), + _ => None, + }; + // Types an untyped object default that is not a plain literal (`{ a: x }`). + let literal_shape = match &*right { + Expr::Object(_) => Some(Typ::Object(ObjectType::new(None, None))), + _ => None, + }; let dflt = if skip_dflt { None } else { @@ -593,14 +604,19 @@ fn parse_param( Expr::Lit(Lit::Num(Number { value, .. })) => Some(serde_json::json!(value)), Expr::Lit(Lit::BigInt(BigInt { value, .. })) => Some(serde_json::json!(value)), Expr::Lit(Lit::Bool(Bool { value, .. })) => Some(Value::Bool(value)), - Expr::Object(ObjectLit { span, .. }) => eval_span(span, cm), - Expr::Array(ArrayLit { span, .. }) => eval_span(span, cm), + Expr::Object(ObjectLit { span, .. }) | Expr::Array(ArrayLit { span, .. }) => { + literal.or_else(|| eval_span(span, cm)) + } _ => None, } }; - if typ == Typ::Unknown && dflt.is_some() { - typ = json_to_typ(dflt.as_ref().unwrap(), false); + if typ == Typ::Unknown { + match &dflt { + Some(d) => typ = json_to_typ(d, false), + None if !skip_dflt => typ = literal_shape.unwrap_or(Typ::Unknown), + None => {} + } } Ok(Arg { otyp, @@ -636,6 +652,59 @@ fn parse_param( r } +/// A default written as a plain literal, read off the AST: `['a']` and `{ a: 1 }` are +/// not JSON, and outside wasm nothing can evaluate them (see `eval_sync`). +fn literal_json(expr: &Expr) -> Option { + let num = |v: f64| { + if v.fract() == 0.0 && v.abs() < 9007199254740992.0 { + Some(serde_json::json!(v as i64)) + } else { + serde_json::Number::from_f64(v).map(Value::Number) + } + }; + match expr { + Expr::Lit(Lit::Str(s)) => Some(Value::String(s.value.to_string())), + Expr::Lit(Lit::Num(n)) => num(n.value), + Expr::Lit(Lit::Bool(b)) => Some(Value::Bool(b.value)), + Expr::Lit(Lit::Null(_)) => Some(Value::Null), + Expr::Unary(UnaryExpr { op: UnaryOp::Minus, arg, .. }) => match &**arg { + Expr::Lit(Lit::Num(n)) => num(-n.value), + _ => None, + }, + Expr::Paren(p) => literal_json(&p.expr), + Expr::Array(a) => a + .elems + .iter() + .map(|e| match e { + Some(ExprOrSpread { spread: None, expr }) => literal_json(expr), + _ => None, + }) + .collect::>>() + .map(Value::Array), + Expr::Object(o) => o + .props + .iter() + .map(|p| match p { + PropOrSpread::Prop(p) => match &**p { + Prop::KeyValue(KeyValueProp { key, value }) => { + let key = match key { + PropName::Ident(i) => i.sym.to_string(), + PropName::Str(s) => s.value.to_string(), + PropName::Num(n) => n.value.to_string(), + _ => return None, + }; + Some((key, literal_json(value)?)) + } + _ => None, + }, + _ => None, + }) + .collect::>>() + .map(Value::Object), + _ => None, + } +} + fn eval_span(span: Span, cm: &Lrc) -> Option { let expr = cm .span_to_snippet(span) @@ -1361,5 +1430,7 @@ pub fn eval_sync(code: &str) -> Result { #[cfg(not(target_arch = "wasm32"))] pub fn eval_sync(_code: &str) -> Result { - panic!("eval_sync is only available in wasm32") + // The server parses signatures with defaults too: a default that is not JSON + // (`{ a: 1 }`, `['x']`) is then left out rather than failing the parse. + Err("evaluating a JavaScript default needs the wasm build".to_string()) } diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 4206dd68f8..b4abc23488 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -7,6 +7,38 @@ mod tests { remove_pinned_import_specifiers, }; + /// The server parses defaults natively, where a non-JSON literal cannot be evaluated + /// but must still type and default the argument as the editor's wasm parse does. + #[test] + fn non_json_literal_defaults_parse_natively() { + let code = + "export function main(opts = { enabled: true, 'n': -1 }, tags = ['a'], o = { a: x }) {}"; + let sig = parse_deno_signature(code, false, false, None).unwrap(); + let defaults: Vec<_> = sig + .args + .iter() + .map(|a| (a.typ.clone(), a.default.clone())) + .collect(); + assert_eq!( + defaults, + vec![ + ( + Typ::Object(ObjectType::new( + None, + Some(vec![ + ObjectProperty::new("enabled".into(), Box::new(Typ::Bool)), + ObjectProperty::new("n".into(), Box::new(Typ::Int)), + ]) + )), + Some(json!({"enabled": true, "n": -1})) + ), + (Typ::List(Box::new(Typ::Str(None))), Some(json!(["a"]))), + // Not a plain literal: typed by its shape, with no default value. + (Typ::Object(ObjectType::new(None, None)), None), + ] + ); + } + #[test] fn test_imports_basic() { let code = r#" diff --git a/backend/parsers/windmill-parser/src/json_schema.rs b/backend/parsers/windmill-parser/src/json_schema.rs new file mode 100644 index 0000000000..6ca216dd89 --- /dev/null +++ b/backend/parsers/windmill-parser/src/json_schema.rs @@ -0,0 +1,402 @@ +use serde_json::{json, Map, Value}; + +use crate::{MainArgSignature, ObjectProperty, Typ}; + +/// The run-form schema of a script's `main`, derived the way the editor derives it +/// (`inferArgs` in the frontend, `argSigToJsonSchemaType` in `windmill-utils-internal`), +/// so a script deployed without a schema gets the one the editor would have sent. +/// `previous` is the schema of the version being superseded: as in the editor, it is +/// updated in place, so what the user set on it (argument order, descriptions, ...) +/// survives wherever the code still agrees with it. +pub fn main_arg_schema(sig: &MainArgSignature, previous: Option<&Value>) -> Value { + let mut schema = match previous { + Some(Value::Object(p)) => p.clone(), + _ => json!({ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + }) + .as_object() + .cloned() + .unwrap_or_default(), + }; + let mut old_props = match schema.remove("properties") { + Some(Value::Object(p)) => p, + _ => Map::new(), + }; + let mut properties = Map::new(); + let mut required = vec![]; + for arg in &sig.args { + let mut prop = match old_props.remove(&arg.name) { + Some(Value::Object(p)) => p, + _ => fresh_property(), + }; + if !matches!(arg.typ, Typ::OneOf(_)) { + prop.remove("oneOf"); + } + apply_typ(&arg.typ, &mut prop); + // `T | T[]` marks an argument debouncing accumulates into. + if let Some(otyp) = arg + .otyp + .as_deref() + .filter(|o| o.contains('[') && o.contains('|')) + { + prop.insert("originalType".into(), json!(otyp)); + } + match &arg.default { + Some(d) => prop.insert("default".into(), d.clone()), + None => prop.remove("default"), + }; + if !arg.has_default { + required.push(json!(arg.name)); + } + properties.insert(arg.name.clone(), Value::Object(prop)); + } + schema.insert("properties".into(), Value::Object(properties)); + schema.insert("required".into(), Value::Array(required)); + if sig.has_cmd_binding == Some(true) { + schema.insert("x-windmill-ps-cmd-binding".into(), json!(true)); + schema.insert( + "x-windmill-ps-supports-should-process".into(), + json!(sig.supports_should_process.unwrap_or(false)), + ); + } else { + schema.remove("x-windmill-ps-cmd-binding"); + schema.remove("x-windmill-ps-supports-should-process"); + } + Value::Object(schema) +} + +/// Keys a user sets on a property in the schema editor, kept whatever the code says. +const USER_FIELDS: &[&str] = &[ + "description", + "pattern", + "min", + "max", + "currency", + "currencyLocale", + "multiselect", + "customErrorMessage", + "required", + "showExpr", + "password", + "order", + "dateFormat", + "title", + "placeholder", +]; + +/// Item keys a list the parser cannot see into keeps from the previous schema. +const ITEMS_USER_FIELDS: &[&str] = &[ + "properties", + "required", + "additionalProperties", + "enum", + "resourceType", + "contentEncoding", + "description", +]; + +fn fresh_property() -> Map { + let mut p = Map::new(); + p.insert("description".into(), json!("")); + p.insert("type".into(), json!("")); + p +} + +fn object_properties( + props: &[ObjectProperty], + old: Option<&Map>, +) -> Map { + props + .iter() + .map(|p| { + let mut prop = old + .and_then(|o| o.get(&p.key)) + .and_then(Value::as_object) + .cloned() + .unwrap_or_else(fresh_property); + apply_typ(&p.typ, &mut prop); + (p.key.clone(), Value::Object(prop)) + }) + .collect() +} + +fn kept_items(old: &Map) -> Value { + let old_items = old.get("items").and_then(Value::as_object); + let mut items = Map::new(); + items.insert( + "type".into(), + old_items + .and_then(|i| i.get("type")) + .filter(|t| t.as_str().is_some_and(|t| !t.is_empty())) + .cloned() + .unwrap_or(json!("object")), + ); + if let Some(old_items) = old_items { + for f in ITEMS_USER_FIELDS { + if let Some(v) = old_items.get(*f) { + items.insert(f.to_string(), v.clone()); + } + } + } + Value::Object(items) +} + +/// Port of `argSigToJsonSchemaType`: rewrites `old` in place to carry `typ`. +fn apply_typ(typ: &Typ, old: &mut Map) { + let mut new = Map::new(); + // Keys the editor sets to `undefined`, which drops them from the stored JSON. + let mut unset: Vec<&str> = vec![]; + let mut set = |k: &str, v: Value| { + new.insert(k.to_string(), v); + }; + match typ { + Typ::Int => set("type", json!("integer")), + Typ::Float => set("type", json!("number")), + Typ::Bool => set("type", json!("boolean")), + Typ::Email => { + set("type", json!("string")); + set("format", json!("email")); + } + Typ::Sql => { + set("type", json!("string")); + set("format", json!("sql")); + } + Typ::Bytes => { + set("type", json!("string")); + set("contentEncoding", json!("base64")); + set("originalType", json!("bytes")); + } + Typ::Datetime => { + set("type", json!("string")); + set("format", json!("date-time")); + } + Typ::Date => { + set("type", json!("string")); + set("format", json!("date")); + } + Typ::OneOf(variants) => { + set("type", json!("object")); + let old_variants = old.get("oneOf").and_then(Value::as_array); + let one_of: Vec = variants + .iter() + .map(|v| { + let old_variant = old_variants + .and_then(|o| o.iter().find(|o| o.get("title") == Some(&json!(v.label)))) + .and_then(Value::as_object); + let old_props = old_variant + .and_then(|o| o.get("properties")) + .and_then(Value::as_object); + let mut variant = json!({ + "type": "object", + "title": v.label, + "properties": object_properties(&v.properties, old_props), + }); + if let Some(order) = old_variant.and_then(|o| o.get("order")) { + variant["order"] = order.clone(); + } + variant + }) + .collect(); + set("oneOf", Value::Array(one_of)); + } + Typ::Object(o) => { + set("type", json!("object")); + if let Some(name) = &o.name { + set("format", json!(format!("resource-{name}"))); + } + if let Some(props) = &o.props { + let old_props = old.get("properties").and_then(Value::as_object); + set( + "properties", + Value::Object(object_properties(props, old_props)), + ); + } + } + Typ::Str(variants) => { + set("type", json!("string")); + match variants { + Some(v) => { + set("originalType", json!("enum")); + set("enum", json!(v)); + } + None => { + set("originalType", json!("string")); + match old.get("enum") { + Some(e) if old.get("originalType") == Some(&json!("string")) => { + set("enum", e.clone()) + } + _ => unset.push("enum"), + } + } + } + } + Typ::Resource(name) => { + set("type", json!("object")); + set("format", json!(format!("resource-{name}"))); + } + Typ::DynSelect(name) => { + set("type", json!("object")); + set("format", json!(format!("dynselect-{name}"))); + } + Typ::DynMultiselect(name) => { + set("type", json!("object")); + set("format", json!(format!("dynmultiselect-{name}"))); + } + Typ::List(inner) => { + set("type", json!("array")); + let (items, original_type) = match inner.as_ref() { + Typ::Int | Typ::Float => (json!({"type": "number"}), "number[]"), + Typ::Bytes => ( + json!({"type": "string", "contentEncoding": "base64"}), + "bytes[]", + ), + Typ::Str(Some(v)) => (json!({"type": "string", "enum": v}), "enum[]"), + Typ::Str(None) => { + let mut items = json!({"type": "string"}); + if let Some(e) = old.get("items").and_then(|i| i.get("enum")) { + items["enum"] = e.clone(); + } + (items, "string[]") + } + Typ::Resource(r) => (json!({"type": "resource", "resourceType": r}), "resource[]"), + Typ::Object(o) => { + if let Some(name) = &o.name { + set("format", json!(format!("resource-{name}"))); + } + let items = match o.props.as_deref().filter(|p| !p.is_empty()) { + Some(props) => json!({ + "type": "object", + "properties": object_properties(props, None), + }), + None => kept_items(old), + }; + (items, "record[]") + } + _ => (kept_items(old), "object[]"), + }; + set("items", items); + set("originalType", json!(original_type)); + } + // The parser cannot tell: whatever type the user gave the argument stands. + Typ::Unknown => set("type", old.get("type").cloned().unwrap_or(json!("object"))), + } + + for f in USER_FIELDS { + if let Some(v) = old.get(*f) { + new.insert(f.to_string(), v.clone()); + } + } + + let old_type = old.get("type").cloned(); + let new_type = new.get("type").cloned(); + let old_format = old + .get("format") + .and_then(Value::as_str) + .map(str::to_string); + let new_format = new + .get("format") + .and_then(Value::as_str) + .map(str::to_string); + let is_date = |f: &Option| matches!(f.as_deref(), Some("date" | "date-time")); + let mut keep_old_format = false; + // A changed type takes every key `new` sets, so none of the adjustments below + // apply: running them would carry the old type's date format or items over. + if old_type != new_type { + } else if new_format.as_deref() == Some("date-time") && old_format.as_deref() == Some("date") { + new.insert("format".into(), json!("date")); + } else if is_date(&new_format) { + keep_old_format = true; + } else if old.get("items").and_then(|i| i.get("type")) + != new.get("items").and_then(|i| i.get("type")) + { + old.remove("items"); + } else if old_type == Some(json!("string")) && old_format.is_some() { + keep_old_format = true; + } + if keep_old_format { + match old.get("format") { + Some(f) => { + new.insert("format".into(), f.clone()); + } + None => { + new.remove("format"); + unset.push("format"); + } + } + } + if (old_type != new_type || new_type != Some(json!("string"))) && !new.contains_key("format") { + old.remove("format"); + } + + for k in unset { + if !new.contains_key(k) { + old.remove(k); + } + } + old.extend(new); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Arg, ObjectType}; + + fn arg(name: &str, typ: Typ, default: Option) -> Arg { + Arg { + name: name.to_string(), + has_default: default.is_some(), + default, + typ, + ..Default::default() + } + } + + #[test] + fn keeps_what_the_user_set_on_the_previous_version() { + let sig = MainArgSignature { + args: vec![ + arg("n", Typ::Int, None), + arg("db", Typ::Resource("postgresql".into()), None), + arg( + "tags", + Typ::List(Box::new(Typ::Str(None))), + Some(json!(["a"])), + ), + arg( + "opts", + Typ::Object(ObjectType::new(None, None)), + Some(json!({})), + ), + arg("any", Typ::Unknown, None), + ], + ..Default::default() + }; + let previous = json!({ + "type": "object", + "order": ["db", "n"], + "properties": { + "n": {"type": "integer", "description": "how many", "min": 1, "nullable": true}, + "tags": {"type": "array", "items": {"type": "string", "enum": ["a", "b"]}}, + "gone": {"type": "string"}, + }, + }); + let schema = main_arg_schema(&sig, Some(&previous)); + assert_eq!(schema["order"], json!(["db", "n"])); + assert_eq!(schema["required"], json!(["n", "db", "any"])); + assert_eq!( + schema["properties"]["n"], + json!({"type": "integer", "description": "how many", "min": 1, "nullable": true}) + ); + assert!(schema["properties"].get("gone").is_none()); + assert_eq!(schema["properties"]["db"]["format"], "resource-postgresql"); + assert_eq!( + schema["properties"]["tags"]["items"], + json!({"type": "string", "enum": ["a", "b"]}) + ); + assert_eq!(schema["properties"]["tags"]["default"], json!(["a"])); + assert_eq!(schema["properties"]["opts"]["type"], "object"); + // Untyped in the code and new: any JSON value, as the editor stores it. + assert_eq!(schema["properties"]["any"]["type"], ""); + } +} diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 4270edcc79..7ca226b432 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -15,6 +15,7 @@ use serde_json::Value; pub mod asset_parser; pub mod duckdb_builtins; pub mod duckdb_macros; +pub mod json_schema; pub mod sql_materialize; /// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) diff --git a/backend/tests/script_schema_inference.rs b/backend/tests/script_schema_inference.rs new file mode 100644 index 0000000000..083716f8df --- /dev/null +++ b/backend/tests/script_schema_inference.rs @@ -0,0 +1,115 @@ +use std::collections::HashMap; + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use windmill_api_client::types::{NewScript, ScriptLang}; +use windmill_test_utils::init_client; + +const PATH: &str = "u/test-user/inferred_schema"; + +fn bun_ns(content: &str, parent_hash: Option) -> NewScript { + NewScript { + draft_only: None, + content: content.into(), + language: ScriptLang::Bun, + lock: None, + parent_hash, + path: PATH.into(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + envs: vec![], + is_template: None, + kind: None, + summary: "".to_string(), + tag: None, + // What this client sends for "no schema". + schema: HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_secs: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + auto_kind: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + assets: vec![], + modules: None, + } +} + +async fn head(db: &Pool) -> anyhow::Result<(i64, Value)> { + Ok(sqlx::query_as( + "SELECT hash, schema::jsonb FROM script \ + WHERE workspace_id = 'test-workspace' AND path = $1 AND archived = false", + ) + .bind(PATH) + .fetch_one(db) + .await?) +} + +/// A deploy without a schema (MCP, a bare API call) gets the one the editor would +/// have sent, and a new version keeps what the user wrote on the previous one's. +#[sqlx::test(fixtures("base"))] +async fn schemaless_deploy_infers_schema_keeping_annotations( + db: Pool, +) -> anyhow::Result<()> { + let (client, _port, _s) = init_client(db.clone()).await; + + client + .create_script( + "test-workspace", + &bun_ns( + "export function main(n: number, name = \"x\") { return n }", + None, + ), + ) + .await?; + let (hash, schema) = head(&db).await?; + assert_eq!(schema["required"], json!(["n"])); + assert_eq!(schema["properties"]["n"]["type"], "number"); + assert_eq!(schema["properties"]["name"]["default"], "x"); + + sqlx::query( + "UPDATE script SET schema = jsonb_set(schema::jsonb, '{properties,n,description}', \ + '\"how many\"')::json WHERE hash = $1", + ) + .bind(hash) + .execute(&db) + .await?; + + client + .create_script( + "test-workspace", + &bun_ns( + "export function main(n: number, flag: boolean) { return n }", + Some(format!("{hash:016x}")), + ), + ) + .await?; + let (hash, schema) = head(&db).await?; + assert_eq!(schema["required"], json!(["n", "flag"])); + assert_eq!(schema["properties"]["n"]["description"], "how many"); + assert_eq!(schema["properties"]["flag"]["type"], "boolean"); + assert!(schema["properties"].get("name").is_none()); + + // Code that does not parse keeps the schema rather than losing it for good. + client + .create_script( + "test-workspace", + &bun_ns( + "export function main(n: number {", + Some(format!("{hash:016x}")), + ), + ) + .await?; + assert_eq!(head(&db).await?.1, schema); + + Ok(()) +} diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index 8b34512872..72353c6f1b 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -30,7 +30,11 @@ windmill-parser-ts-asset.workspace = true windmill-parser-sql-asset.workspace = true windmill-parser-sql.workspace = true windmill-parser-yaml.workspace = true +windmill-parser-bash.workspace = true +windmill-parser-go.workspace = true +windmill-parser-graphql.workspace = true +anyhow.workspace = true axum.workspace = true futures.workspace = true http.workspace = true diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 51f2619805..631e8606e6 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1132,6 +1132,96 @@ async fn validate_dbt_relation( }) } +/// The schema the editor would have derived from `content`, for a deploy that sends +/// none (MCP, a bare API call). `None` for a language whose parser this build lacks, +/// or code that does not parse. +fn infer_main_schema( + language: &ScriptLang, + kind: Option<&ScriptKind>, + content: &str, + previous: Option<&serde_json::Value>, +) -> Option { + let entrypoint = + matches!(kind, Some(ScriptKind::Preprocessor)).then(|| "preprocessor".to_string()); + // The database a SQL script runs against is an argument unless the code names + // it (`-- database f/...`), as in the editor. + let with_db = |sig: anyhow::Result, resource: &str| { + sig.map(|mut sig| { + if windmill_parser_sql::parse_db_resource(content).is_none() { + sig.args.insert( + 0, + windmill_parser::Arg { + name: "database".to_string(), + typ: windmill_parser::Typ::Resource(resource.to_string()), + ..Default::default() + }, + ); + } + sig + }) + }; + let sig = match language { + ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { + windmill_parser_ts::parse_deno_signature(content, false, false, entrypoint) + } + #[cfg(feature = "python")] + ScriptLang::Python3 => { + windmill_parser_py::parse_python_signature(content, entrypoint, false) + } + ScriptLang::Go => windmill_parser_go::parse_go_sig(content), + ScriptLang::Bash => windmill_parser_bash::parse_bash_sig(content), + ScriptLang::Powershell => windmill_parser_bash::parse_powershell_sig(content), + ScriptLang::Postgresql => { + with_db(windmill_parser_sql::parse_pgsql_sig(content), "postgresql") + } + ScriptLang::Mysql => with_db(windmill_parser_sql::parse_mysql_sig(content), "mysql"), + ScriptLang::Bigquery => { + with_db(windmill_parser_sql::parse_bigquery_sig(content), "bigquery") + } + ScriptLang::Snowflake => with_db( + windmill_parser_sql::parse_snowflake_sig(content), + "snowflake", + ), + ScriptLang::Mssql => with_db( + windmill_parser_sql::parse_mssql_sig(content), + "ms_sql_server", + ), + ScriptLang::OracleDB => { + with_db(windmill_parser_sql::parse_oracledb_sig(content), "oracledb") + } + ScriptLang::DuckDb => windmill_parser_sql::parse_duckdb_sig(content), + ScriptLang::Graphql => { + windmill_parser_graphql::parse_graphql_sig(content).map(|mut sig| { + sig.args.insert( + 0, + windmill_parser::Arg { + name: "api".to_string(), + typ: windmill_parser::Typ::Resource("graphql".to_string()), + ..Default::default() + }, + ); + sig + }) + } + ScriptLang::Ansible => windmill_parser_yaml::parse_ansible_sig(content), + _ => return None, + }; + match sig { + Ok(sig) => serde_json::value::to_raw_value(&windmill_parser::json_schema::main_arg_schema( + &sig, previous, + )) + .ok() + .map(|v| Schema(sqlx::types::Json(v))), + Err(e) => { + tracing::warn!( + "could not infer the schema of a {} script: {e:#}", + language.as_str() + ); + None + } + } +} + async fn create_script_internal<'c>( mut ns: NewScript, w_id: String, @@ -1359,6 +1449,40 @@ async fn create_script_internal<'c>( parent_adopted_from_retired_path = ns.parent_hash.is_some(); } + // Before hashing, so the hash and the no-op check see the schema that gets stored. + // `{}` counts as absent: an agent filling every tool argument sends it for "none". + let schema_absent = ns.schema.as_ref().is_none_or(|s| { + serde_json::from_str::>(s.0.get()) + .is_ok_and(|m| m.is_empty()) + }); + if schema_absent && !matches!(ns.language, ScriptLang::Dbt) { + let previous = match &ns.parent_hash { + Some(p_hash) => sqlx::query_scalar::<_, Option>>( + "SELECT schema FROM script WHERE hash = $1 AND workspace_id = $2", + ) + .bind(p_hash.0) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + .flatten() + .map(|s| s.0), + None => None, + }; + // Code the server cannot parse keeps the previous version's schema, as it + // does in the editor, rather than losing it for good. + ns.schema = infer_main_schema( + &ns.language, + ns.kind.as_ref(), + &ns.content, + previous.as_ref(), + ) + .or_else(|| { + previous + .and_then(|p| serde_json::value::to_raw_value(&p).ok()) + .map(|v| Schema(sqlx::types::Json(v))) + }); + } + // Must stay below the parent resolution above: an auto_parent deploy hashed before // it carries a first deploy's lineage, so redeploying content the path has held // before collides with that archived version instead of superseding it. The diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 016342a6fe..4346dfcd0a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10685,6 +10685,9 @@ paths: - kind - tag - deployment_message + - schema + x-mcp-tool-opaque-fields: + - schema tags: - script parameters: @@ -10724,6 +10727,9 @@ paths: - kind - tag - deployment_message + - schema + x-mcp-tool-opaque-fields: + - schema tags: - script parameters: @@ -29742,6 +29748,15 @@ components: type: string schema: type: object + description: >- + JSON Schema of the arguments of `main`, which is what a run form and an + MCP tool offer. Omitted (or `{}`), it is inferred from `content` for + TypeScript, Python, Go, Bash, PowerShell, SQL, GraphQL and Ansible + scripts. For other languages, or code that does not parse, a new script + gets none. A new version of an existing script also keeps what the + previous version's schema says about each argument, or that whole schema + when nothing can be inferred. A dbt script always takes its schema from its + descriptor, whatever is sent. is_template: type: boolean lock: diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index ffefaba2dd..1962710578 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -699,6 +699,10 @@ pub fn all_tools() -> Vec { "content": { "type": "string" }, + "schema": { + "type": "object", + "description": "JSON Schema of the arguments of `main`, which is what a run form and an MCP tool offer. Omitted (or `{}`), it is inferred from `content` for TypeScript, Python, Go, Bash, PowerShell, SQL, GraphQL and Ansible scripts. For other languages, or code that does not parse, a new script gets none. A new version of an existing script also keeps what the previous version's schema says about each argument, or that whole schema when nothing can be inferred. A dbt script always takes its schema from its descriptor, whatever is sent." + }, "language": { "type": "string", "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" @@ -757,6 +761,10 @@ is, a different one moves it there and archives the old path"), "content": { "type": "string" }, + "schema": { + "type": "object", + "description": "JSON Schema of the arguments of `main`, which is what a run form and an MCP tool offer. Omitted (or `{}`), it is inferred from `content` for TypeScript, Python, Go, Bash, PowerShell, SQL, GraphQL and Ansible scripts. For other languages, or code that does not parse, a new script gets none. A new version of an existing script also keeps what the previous version's schema says about each argument, or that whole schema when nothing can be inferred. A dbt script always takes its schema from its descriptor, whatever is sent." + }, "language": { "type": "string", "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" @@ -1024,10 +1032,12 @@ is, a different one moves it there and archives the old path"), "description": "Detailed documentation for this flow" }, "value": { - "type": "object" + "type": "object", + "description": "The flow structure containing modules and optional preprocessor/failure handlers" }, "schema": { - "type": "object" + "type": "object", + "description": "JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')" }, "path": { "type": "string" @@ -1080,10 +1090,12 @@ is, a different one moves it there and archives the old path"), "description": "Detailed documentation for this flow" }, "value": { - "type": "object" + "type": "object", + "description": "The flow structure containing modules and optional preprocessor/failure handlers" }, "schema": { - "type": "object" + "type": "object", + "description": "JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')" }, "tag": { "type": "string" diff --git a/frontend/src/lib/mcpEndpointTools.ts b/frontend/src/lib/mcpEndpointTools.ts index a9e2b9f3b2..2197998857 100644 --- a/frontend/src/lib/mcpEndpointTools.ts +++ b/frontend/src/lib/mcpEndpointTools.ts @@ -708,6 +708,10 @@ export const mcpEndpointTools: EndpointTool[] = [ "content": { "type": "string" }, + "schema": { + "type": "object", + "description": "JSON Schema of the arguments of `main`, which is what a run form and an MCP tool offer. Omitted (or `{}`), it is inferred from `content` for TypeScript, Python, Go, Bash, PowerShell, SQL, GraphQL and Ansible scripts. For other languages, or code that does not parse, a new script gets none. A new version of an existing script also keeps what the previous version's schema says about each argument, or that whole schema when nothing can be inferred. A dbt script always takes its schema from its descriptor, whatever is sent." + }, "language": { "type": "string", "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" @@ -764,6 +768,10 @@ export const mcpEndpointTools: EndpointTool[] = [ "content": { "type": "string" }, + "schema": { + "type": "object", + "description": "JSON Schema of the arguments of `main`, which is what a run form and an MCP tool offer. Omitted (or `{}`), it is inferred from `content` for TypeScript, Python, Go, Bash, PowerShell, SQL, GraphQL and Ansible scripts. For other languages, or code that does not parse, a new script gets none. A new version of an existing script also keeps what the previous version's schema says about each argument, or that whole schema when nothing can be inferred. A dbt script always takes its schema from its descriptor, whatever is sent." + }, "language": { "type": "string", "description": "Possible values: python3, deno, go, bash, powershell, postgresql, mysql, bigquery, snowflake, mssql, oracledb, graphql, nativets, bun, php, rust, ansible, csharp, nu, java, ruby, rlang, duckdb, bunnative, dbt" @@ -1031,10 +1039,12 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "Detailed documentation for this flow" }, "value": { - "type": "object" + "type": "object", + "description": "The flow structure containing modules and optional preprocessor/failure handlers" }, "schema": { - "type": "object" + "type": "object", + "description": "JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')" }, "path": { "type": "string" @@ -1087,10 +1097,12 @@ export const mcpEndpointTools: EndpointTool[] = [ "description": "Detailed documentation for this flow" }, "value": { - "type": "object" + "type": "object", + "description": "The flow structure containing modules and optional preprocessor/failure handlers" }, "schema": { - "type": "object" + "type": "object", + "description": "JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')" }, "tag": { "type": "string" From bc4f872f1093a59b9b8fbc3e11a6f00b42474f0c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 25 Sep 2026 12:30:43 +0200 Subject: [PATCH 16/91] perf: skip the flow_env ancestor walk for sub-flows with nothing to inherit (#11349) * perf: skip the flow_env ancestor walk for sub-flows with nothing to inherit Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep the flow_env mark out of sub-flow definitions replayed by restarts Co-Authored-By: Claude Opus 5.5 (1M context) * test: align the guest scopable-path test with app path validation Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/tests/app_guest_execution_mode.rs | 27 ++--- backend/tests/flow_engine_parity.rs | 125 +++++++++++++++++++++ backend/tests/job_payload.rs | 1 + backend/windmill-queue/src/jobs.rs | 6 +- backend/windmill-types/src/flow_status.rs | 6 + backend/windmill-types/src/jobs.rs | 2 + backend/windmill-worker/src/worker_flow.rs | 86 +++++++++----- 7 files changed, 209 insertions(+), 44 deletions(-) diff --git a/backend/tests/app_guest_execution_mode.rs b/backend/tests/app_guest_execution_mode.rs index 7af2e0794e..105458847a 100644 --- a/backend/tests/app_guest_execution_mode.rs +++ b/backend/tests/app_guest_execution_mode.rs @@ -577,24 +577,13 @@ async fn guests_mode_needs_a_scopable_path(db: Pool) -> anyhow::Result .send() .await?; assert_eq!(resp.status(), 400, "renamed to a `,` path while in Guests"); - let resp = authed( - client().post(format!("{ws}/apps/update/{APP_PATH}")), - ADMIN_TOKEN, - ) - .json(&json!({ "path": "u/test-user/My App" })) - .send() - .await?; - assert_eq!( - resp.status(), - 200, - "a space is literal: {}", - resp.text().await? - ); - // Set on update: an app that already sits on such a path cannot be switched. + // Set on update: an app that already sits on such a path cannot be switched. Path + // validation now refuses these paths on every write, so plant the kind of app saved + // before it did. let resp = authed(client().post(format!("{ws}/apps/create")), ADMIN_TOKEN) .json(&json!({ - "path": "u/test-user/x:y", + "path": "u/test-user/legacy", "summary": "App", "value": {}, "policy": { "execution_mode": "publisher", "triggerables_v2": {} } @@ -602,6 +591,12 @@ async fn guests_mode_needs_a_scopable_path(db: Pool) -> anyhow::Result .send() .await?; assert_eq!(resp.status(), 201, "{}", resp.text().await?); + sqlx::query( + "UPDATE app SET path = 'u/test-user/x:y' + WHERE workspace_id = 'test-workspace' AND path = 'u/test-user/legacy'", + ) + .execute(&db) + .await?; let resp = authed( client().post(format!("{ws}/apps/update/u/test-user/x:y")), ADMIN_TOKEN, @@ -610,6 +605,8 @@ async fn guests_mode_needs_a_scopable_path(db: Pool) -> anyhow::Result .send() .await?; assert_eq!(resp.status(), 400, "switched to Guests on a `:` path"); + let body = resp.text().await?; + assert!(body.contains("cannot be set to Guests"), "{body}"); Ok(()) } diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index b6a943a23b..7ea221c2e8 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -2877,6 +2877,131 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a Ok(()) } +// A branch/loop sub-flow of a deployed flow is marked at push when its parent has no flow_env, +// own or inherited, so its worker skips `fetch_root_flow_env`. With an env two scopes up (root +// → loop → imported flow without env → parallel loop → leaf) nothing may be marked and the +// root's value must reach the leaf; with no env anywhere every loop iteration is marked. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +async fn test_flow_env_marks_sub_flows_only_without_ancestor_env( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Deployed, so their loops run as `FlowNode` sub-flows. + let deploy = |path: &'static str, version: i64, value: serde_json::Value| { + let db = db.clone(); + async move { + sqlx::query( + "INSERT INTO flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ('test-workspace', '', '', $1, ARRAY[$2]::bigint[], '{}'::jsonb, $3, 'system')", + ) + .bind(path) + .bind(version) + .bind(&value) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO flow_version(id, workspace_id, path, schema, value, created_by) VALUES ($1, 'test-workspace', $2, '{}'::jsonb, $3, 'system')", + ) + .bind(version) + .bind(path) + .bind(&value) + .execute(&db) + .await?; + RunJob::from(JobPayload::FlowDependencies { + path: path.to_string(), + dedicated_worker: None, + version, + debouncing_settings: Default::default(), + }) + .run_until_complete(&db, false, port) + .await; + anyhow::Ok(()) + } + }; + deploy( + "f/system/imported_loop_no_env", + 9991004, + json!({ "modules": [{ "id": "inner", "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "[1, 2]" }, + "skip_failures": false, + "parallel": true, + "modules": [{ "id": "leaf", "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main(key: any) { return key ?? null; }", + "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } } + }}, { "id": "tail", "value": { + // A single simple step is inlined rather than stored as a flow node. + "type": "rawscript", + "language": "deno", + "content": "export function main(prev: any) { return prev; }", + "input_transforms": { "prev": { "type": "javascript", "expr": "results.leaf" } } + }}] + }}]}), + ) + .await?; + let root = |flow_env: serde_json::Value| { + json!({ "flow_env": flow_env, "modules": [{ "id": "outer", "value": { + "type": "forloopflow", + "iterator": { "type": "javascript", "expr": "[1]" }, + "skip_failures": false, + "parallel": false, + "modules": [{ "id": "import", "value": { + "type": "flow", + "path": "f/system/imported_loop_no_env", + "input_transforms": {} + }}] + }}]}) + }; + deploy( + "f/system/root_with_env", + 9991005, + root(json!({ "KEY": "from-root" })), + ) + .await?; + deploy("f/system/root_no_env", 9991006, root(json!(null))).await?; + + let run = |path: &'static str, version: i64| { + let db = db.clone(); + async move { + let completed = RunJob::from(JobPayload::Flow { + path: path.to_string(), + dedicated_worker: None, + apply_preprocessor: false, + version, + labels: None, + }) + .run_until_complete(&db, false, port) + .await; + // The mark of every loop iteration pushed under the run. + let marks = sqlx::query_as::<_, (String, Option)>( + "SELECT j.kind::text, (c.flow_status->>'no_inherited_flow_env')::bool + FROM v2_job j JOIN v2_job_completed c ON c.id = j.id + WHERE COALESCE(j.root_job, j.parent_job) = $1 + AND j.flow_step_id IN ('outer', 'inner')", + ) + .bind(completed.id) + .fetch_all(&db) + .await?; + anyhow::Ok((completed.json_result().unwrap(), marks)) + } + }; + + let (result, marks) = run("f/system/root_with_env", 9991005).await?; + assert_eq!(result, json!([["from-root", "from-root"]])); + assert_eq!(marks, vec![("flownode".to_string(), None); 3]); + + let (result, marks) = run("f/system/root_no_env", 9991006).await?; + assert_eq!(result, json!([[null, null]])); + assert_eq!(marks, vec![("flownode".to_string(), Some(true)); 3]); + + Ok(()) +} + // stop_after_if predicate sees flow_env. Regression for the eval at line 614 // of `update_flow_status_after_job_completion_internal` which used to pass // `None` for flow_env unconditionally. diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index c538cbe6e1..1a973f6fe6 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -239,6 +239,7 @@ mod job_payload { let result = RunJob::from(JobPayload::FlowNode { id, path: "f/system/hello_with_nodes_flow/forloop-0".into(), + no_inherited_flow_env: false, }) .arg("iter", json!({ "value": "tests", "index": 0 })) .run_until_complete(&db, false, port) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 50eb9c8b1f..18fe58becc 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -5854,10 +5854,10 @@ async fn push_inner<'c, 'd>( dedicated_worker, ..Default::default() }, - JobPayload::FlowNode { id, path } => { + JobPayload::FlowNode { id, path, no_inherited_flow_env } => { let data = cache::flow::fetch_flow(db, id).await?; let value = data.value(); - let status = Some(FlowStatus::new(value)); + let status = Some(FlowStatus { no_inherited_flow_env, ..FlowStatus::new(value) }); // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the flow node id. let value_o = if !MIN_VERSION_IS_AT_LEAST_1_440.met().await { @@ -6094,6 +6094,7 @@ async fn push_inner<'c, 'd>( stream_job: None, chat_input_enabled: None, memory_id: None, + no_inherited_flow_env: false, } } _ => { @@ -6488,6 +6489,7 @@ async fn push_inner<'c, 'd>( stream_job: None, chat_input_enabled: None, memory_id: None, + no_inherited_flow_env: false, }; let value = flow_data.value(); let priority = value.priority; diff --git a/backend/windmill-types/src/flow_status.rs b/backend/windmill-types/src/flow_status.rs index 79890e3c6e..6dff8430b7 100644 --- a/backend/windmill-types/src/flow_status.rs +++ b/backend/windmill-types/src/flow_status.rs @@ -41,6 +41,11 @@ pub struct FlowStatus { pub chat_input_enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] pub memory_id: Option, + /// Set at push on a branch/loop sub-flow whose enclosing flow had no `flow_env`, own or + /// inherited, so the worker can skip the ancestor walk for it. `false` only means unknown: + /// the worker then walks the ancestors as usual. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub no_inherited_flow_env: bool, } #[derive(Serialize, Deserialize, Debug, Clone, Default)] @@ -568,6 +573,7 @@ impl FlowStatus { stream_job: None, chat_input_enabled: f.chat_input_enabled, memory_id: None, + no_inherited_flow_env: false, } } diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 5190efe380..4e0c5340f5 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -519,6 +519,8 @@ pub enum JobPayload { FlowNode { id: FlowNodeId, path: String, + /// See [`crate::flow_status::FlowStatus::no_inherited_flow_env`]. + no_inherited_flow_env: bool, }, FlowScript { id: FlowNodeId, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 0ab4e2871c..6e01378448 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -537,15 +537,22 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(cached) = RESOLVED_FLOW_ENV_CACHE.get(&flow) { Some(cached) } else { - resolve_flow_env_for_status_update(db, client, flow, w_id, flow_value) - .await - .map(|(env, is_cacheable)| { - let arc = Arc::new(env); - if is_cacheable { - RESOLVED_FLOW_ENV_CACHE.insert(flow, arc.clone()); - } - arc - }) + resolve_flow_env_for_status_update( + db, + client, + flow, + w_id, + flow_value, + old_status.no_inherited_flow_env, + ) + .await + .map(|(env, is_cacheable)| { + let arc = Arc::new(env); + if is_cacheable { + RESOLVED_FLOW_ENV_CACHE.insert(flow, arc.clone()); + } + arc + }) } } else { None @@ -2521,6 +2528,7 @@ async fn resolve_flow_env_for_status_update( flow_job_id: Uuid, workspace_id: &str, flow_value: &FlowValue, + no_inherited_flow_env: bool, ) -> Option<(HashMap>, bool)> { // Fetch the env source. For the inherited path, we first need to know whether the // flow even has a parent — `fetch_root_flow_env` runs a recursive CTE on `v2_job` @@ -2529,6 +2537,8 @@ async fn resolve_flow_env_for_status_update( let (env, mini): (HashMap>, Option) = if let Some(ref e) = flow_value.flow_env { (e.clone(), None) + } else if no_inherited_flow_env { + return None; } else { let mini = match get_mini_pulled_job(db, &flow_job_id).await { Ok(Some(j)) => j, @@ -2542,7 +2552,13 @@ async fn resolve_flow_env_for_status_update( // No own flow_env and no parent to inherit from — nothing to resolve. return None; } - let env = fetch_root_flow_env(db, flow_job_id, workspace_id).await?; + let env = match fetch_root_flow_env(db, flow_job_id, workspace_id).await { + Ok(env) => env?, + Err(e) => { + tracing::warn!("Failed to look up the inherited flow_env: {e:#}"); + return None; + } + }; (env, Some(mini)) }; if env.is_empty() { @@ -2608,7 +2624,7 @@ async fn fetch_root_flow_env( db: &DB, flow_job_id: Uuid, workspace_id: &str, -) -> Option>> { +) -> Result>>, sqlx::Error> { sqlx::query_scalar!( r#"WITH RECURSIVE chain(id, parent_job, flow_innermost_root_job, runnable_id, runnable_path, raw_flow, depth) AS ( SELECT id, parent_job, flow_innermost_root_job, runnable_id, runnable_path, raw_flow, 0 @@ -2645,10 +2661,7 @@ async fn fetch_root_flow_env( ) .fetch_optional(db) .await - .ok() - .flatten() - .flatten() - .map(|json| json.0) + .map(|row| row.flatten().map(|json| json.0)) } struct FailureContext { @@ -2793,18 +2806,31 @@ pub async fn handle_flow( ) -> anyhow::Result<()> { let flow = flow_data.value(); + let status = flow_job + .parse_flow_status() + .with_context(|| "Unable to parse flow status")?; + // Sub-flows spawned by `payload_from_modules` for branches/loops don't // carry the parent's `flow_env` in their own FlowValue. Fall back to the // nearest enclosing scope's `flow_env` so predicates like `skip_if`, // `stop_after_if`, and branch conditions see the same env as input // transforms. - let inherited_env: Option>> = - if flow.flow_env.is_none() && flow_job.parent_job.is_some() { - fetch_root_flow_env(db, flow_job.id, &flow_job.workspace_id).await - } else { - None - }; + let (inherited_env, inherited_env_known) = if flow.flow_env.is_none() + && flow_job.parent_job.is_some() + && !status.no_inherited_flow_env + { + match fetch_root_flow_env(db, flow_job.id, &flow_job.workspace_id).await { + Ok(env) => (env, true), + Err(e) => { + tracing::warn!("Failed to look up the inherited flow_env: {e:#}"); + (None, false) + } + } + } else { + (None, true) + }; let env_source = flow.flow_env.as_ref().or(inherited_env.as_ref()); + let no_flow_env = inherited_env_known && env_source.is_none_or(|e| e.is_empty()); // Resolve $var: and $res: references in flow_env. // We resolve into a separate variable to avoid cloning the entire FlowValue @@ -2850,10 +2876,6 @@ pub async fn handle_flow( } } - let status = flow_job - .parse_flow_status() - .with_context(|| "Unable to parse flow status")?; - let schedule_path = flow_job.schedule_path(); if !flow_job.is_flow_step() && status.retry.fail_count == 0 @@ -2997,6 +3019,7 @@ pub async fn handle_flow( status, flow, flow_env, + no_flow_env, db, client, last_result.clone(), @@ -3240,6 +3263,8 @@ async fn push_next_flow_job( mut status: FlowStatus, flow: &FlowValue, flow_env: Option<&HashMap>>, + // Neither this flow nor any ancestor defines `flow_env`. + no_flow_env: bool, db: &sqlx::Pool, client: &AuthedClient, last_job_result: Option>>, @@ -4229,7 +4254,14 @@ async fn push_next_flow_job( let mut tx = db.begin().warn_after_seconds(3).await?; let nargs = args.as_ref(); - for (i, payload_tag) in job_payloads.into_iter().enumerate() { + for (i, mut payload_tag) in job_payloads.into_iter().enumerate() { + // A `FlowNode` here is a branch/loop sub-flow from `payload_from_modules`: it never defines + // its own `flow_env`, so it inherits exactly this flow's. The mark lives in the child's + // status, not its definition: a `RawFlow` child's stored definition is replayed by nested + // restarts, whose root may have gained a `flow_env` since. + if let JobPayload::FlowNode { no_inherited_flow_env, .. } = &mut payload_tag.payload { + *no_inherited_flow_env = no_flow_env; + } if i % 100 == 0 && i != 0 { tracing::info!(id = %flow_job.id, root_id = %job_root, "pushed (non-commited yet) first {i} subflows of {len}"); // Ping on the pool, outside `tx`, so the zombie flow monitor sees it before the @@ -5218,7 +5250,7 @@ fn payload_from_modules<'a>( } if let Some(id) = modules_node { - return Some(JobPayload::FlowNode { id, path: path() }); + return Some(JobPayload::FlowNode { id, path: path(), no_inherited_flow_env: false }); } add_virtual_items_if_necessary(&mut modules); From 245628210f7d75aedb9e0a289d1dee63f1513b5b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 25 Sep 2026 12:41:27 +0200 Subject: [PATCH 17/91] perf: skip parent status write when a parallel loop iteration starts (#11348) * perf: skip parent status write when a parallel loop iteration starts Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep the loop viewer off the parent job while a parallel loop runs Co-Authored-By: Claude Opus 5.5 (1M context) * docs: state what a parallel module's job would hold Co-Authored-By: Claude Opus 5.5 (1M context) * perf: test the parallel module with one containment check Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- ...6ceea8981bd3618b90615f7afab360dfbd6fb567b45b.json} | 4 ++-- backend/windmill-queue/src/flow_status.rs | 9 ++++++++- .../src/lib/components/FlowStatusViewerInner.svelte | 11 ++++++++++- 3 files changed, 20 insertions(+), 4 deletions(-) rename backend/.sqlx/{query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json => query-cbb59234f5a2d2e866aa6ceea8981bd3618b90615f7afab360dfbd6fb567b45b.json} (60%) diff --git a/backend/.sqlx/query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json b/backend/.sqlx/query-cbb59234f5a2d2e866aa6ceea8981bd3618b90615f7afab360dfbd6fb567b45b.json similarity index 60% rename from backend/.sqlx/query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json rename to backend/.sqlx/query-cbb59234f5a2d2e866aa6ceea8981bd3618b90615f7afab360dfbd6fb567b45b.json index 3d61a47eb1..f631078e76 100644 --- a/backend/.sqlx/query-5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf.json +++ b/backend/.sqlx/query-cbb59234f5a2d2e866aa6ceea8981bd3618b90615f7afab360dfbd6fb567b45b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n flow_status = jsonb_set(\n jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)),\n ARRAY['modules', $3::INTEGER::TEXT, 'type'],\n to_jsonb('InProgress'::text)\n )\n WHERE id = $2", + "query": "UPDATE v2_job_status SET\n flow_status = jsonb_set(\n jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)),\n ARRAY['modules', $3::INTEGER::TEXT, 'type'],\n to_jsonb('InProgress'::text)\n )\n WHERE id = $2\n AND NOT COALESCE(\n flow_status->'modules'->$3::INTEGER @> '{\"type\": \"InProgress\", \"parallel\": true}',\n false\n )", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "5c141db5cd6806cc27696a38b7f56a190dbea3ff59e2961658dd1677365e0fdf" + "hash": "cbb59234f5a2d2e866aa6ceea8981bd3618b90615f7afab360dfbd6fb567b45b" } diff --git a/backend/windmill-queue/src/flow_status.rs b/backend/windmill-queue/src/flow_status.rs index e6b64c659d..55d2f6ec90 100644 --- a/backend/windmill-queue/src/flow_status.rs +++ b/backend/windmill-queue/src/flow_status.rs @@ -15,6 +15,9 @@ pub async fn update_flow_status_in_progress( let step = get_step_of_flow_status(db, flow).await?; match step { Step::Step { idx: step, .. } => { + // Children of a parallel module start concurrently and are tracked in `flow_jobs`, so its + // `job` would only hold whichever child started last. Writing it rewrites the parent's + // whole flow_status under its row lock once per child, serializing large parallel loops. sqlx::query!( "UPDATE v2_job_status SET flow_status = jsonb_set( @@ -22,7 +25,11 @@ pub async fn update_flow_status_in_progress( ARRAY['modules', $3::INTEGER::TEXT, 'type'], to_jsonb('InProgress'::text) ) - WHERE id = $2", + WHERE id = $2 + AND NOT COALESCE( + flow_status->'modules'->$3::INTEGER @> '{\"type\": \"InProgress\", \"parallel\": true}', + false + )", job_in_progress, flow, step as i32 diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 05ad6c7bb2..1d56c20b38 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -944,6 +944,15 @@ } } + // A running parallel module keeps this flow's own id in `job` (its children are only in + // `flow_jobs`), so passing it through would make the nested viewer watch this flow again. + function innerViewerJobId(mod: FlowStatusModule): string { + if (mod.job && mod.job == job?.id && mod.flow_jobs?.length) { + return mod.flow_jobs[0] + } + return mod.job ?? '' + } + function onJobsLoadedInner(mod: FlowStatusModule, job: Job, force?: boolean): void { let id = mod.id if (id && ((mod.flow_jobs ?? []).length == 0 || force)) { @@ -1783,7 +1792,7 @@ {subflowParentsGlobalModuleStates} {subflowParentsDurationStatuses} {isSelectedBranch} - jobId={mod.job ?? ''} + jobId={innerViewerJobId(mod)} {reducedPolling} innerModule={mod.flow_jobs ? job.raw_flow?.modules[i]?.value : undefined} flowJobIds={mod.flow_jobs From 4b09558e13fadca6fe604c231a5cdfb896d4d0f8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 25 Sep 2026 12:43:46 +0200 Subject: [PATCH 18/91] feat: start a deferred queued job now without changing its id (#11347) * feat: start a deferred queued job now without changing its id Co-Authored-By: Claude Opus 5.5 (1M context) * fix: refuse starting a schedule's upcoming tick early Co-Authored-By: Claude Opus 5.5 (1M context) * fix: hide run now on upcoming schedule ticks and register its audit op Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- ...751e1317e7398dfaf1dfde5c57169cc146c9d.json | 23 ++++ ...b30d71c74fffbba3f5ac6197c985fd73e5c11.json | 23 ++++ ...61af624ab6de67a5c82967f4df8117fe50bc7.json | 47 ++++++++ backend/tests/jobs_read_auth.rs | 93 ++++++++++++++- backend/windmill-api/openapi.yaml | 24 ++++ backend/windmill-api/src/jobs.rs | 112 +++++++++++++++++- .../auditLogs/AuditLogsFilters.svelte | 1 + .../(root)/(logged)/run/[...run]/+page.svelte | 36 +++++- 8 files changed, 354 insertions(+), 5 deletions(-) create mode 100644 backend/.sqlx/query-1740f8bd6f9e084a470a775151e751e1317e7398dfaf1dfde5c57169cc146c9d.json create mode 100644 backend/.sqlx/query-372db51a00e5cf37f00af4b7962b30d71c74fffbba3f5ac6197c985fd73e5c11.json create mode 100644 backend/.sqlx/query-c703da3c3e133536dda9721644561af624ab6de67a5c82967f4df8117fe50bc7.json diff --git a/backend/.sqlx/query-1740f8bd6f9e084a470a775151e751e1317e7398dfaf1dfde5c57169cc146c9d.json b/backend/.sqlx/query-1740f8bd6f9e084a470a775151e751e1317e7398dfaf1dfde5c57169cc146c9d.json new file mode 100644 index 0000000000..46b7d2509c --- /dev/null +++ b/backend/.sqlx/query-1740f8bd6f9e084a470a775151e751e1317e7398dfaf1dfde5c57169cc146c9d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue q SET scheduled_for = now()\n FROM (SELECT id, scheduled_for FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 FOR UPDATE) prev\n WHERE q.id = prev.id AND NOT q.running AND q.suspend = 0 AND q.canceled_by IS NULL\n AND q.scheduled_for > now()\n RETURNING prev.scheduled_for", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scheduled_for", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1740f8bd6f9e084a470a775151e751e1317e7398dfaf1dfde5c57169cc146c9d" +} diff --git a/backend/.sqlx/query-372db51a00e5cf37f00af4b7962b30d71c74fffbba3f5ac6197c985fd73e5c11.json b/backend/.sqlx/query-372db51a00e5cf37f00af4b7962b30d71c74fffbba3f5ac6197c985fd73e5c11.json new file mode 100644 index 0000000000..f5a2ac20ed --- /dev/null +++ b/backend/.sqlx/query-372db51a00e5cf37f00af4b7962b30d71c74fffbba3f5ac6197c985fd73e5c11.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "372db51a00e5cf37f00af4b7962b30d71c74fffbba3f5ac6197c985fd73e5c11" +} diff --git a/backend/.sqlx/query-c703da3c3e133536dda9721644561af624ab6de67a5c82967f4df8117fe50bc7.json b/backend/.sqlx/query-c703da3c3e133536dda9721644561af624ab6de67a5c82967f4df8117fe50bc7.json new file mode 100644 index 0000000000..4ed2afc15b --- /dev/null +++ b/backend/.sqlx/query-c703da3c3e133536dda9721644561af624ab6de67a5c82967f4df8117fe50bc7.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.scheduled_for, s.path, s.schedule, s.cron_version, s.timezone\n FROM v2_job j\n JOIN v2_job_queue q USING (id)\n JOIN schedule s ON s.workspace_id = j.workspace_id AND s.path = j.trigger\n WHERE j.id = $1 AND j.workspace_id = $2 AND j.trigger_kind = 'schedule'\n AND j.parent_job IS NULL AND q.scheduled_for > now()", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "scheduled_for", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "schedule", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "cron_version", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "timezone", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + false + ] + }, + "hash": "c703da3c3e133536dda9721644561af624ab6de67a5c82967f4df8117fe50bc7" +} diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index c8c154c250..8c5bfee0eb 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -516,10 +516,14 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul Some("SECRET_TOKEN_2"), ) .await; - assert!(status.is_success(), "owner must mint a resume secret: {secret}"); + assert!( + status.is_success(), + "owner must mint a resume secret: {secret}" + ); let secret = secret.trim().trim_matches('"').to_string(); - let approval_result = - format!("completed/get_result/{STEP_JOB}?suspended_job={STEP_JOB}&resume_id=0&secret={secret}"); + let approval_result = format!( + "completed/get_result/{STEP_JOB}?suspended_job={STEP_JOB}&resume_id=0&secret={secret}" + ); let (status, body) = get(&base, &approval_result, None).await; assert!( status.is_success(), @@ -991,6 +995,89 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul !status.is_success(), "a public token must not let an anonymous caller cancel the run (got {status}): {body}" ); + // ---- RUN_NOW is gated like cancel, and only moves a job still waiting for a + // future start. ---- + let run_now = format!("queue/run_now/{RUNNING_JOB}"); + let (status, body) = post(&authed_base, &run_now, Some("SECRET_TOKEN_3")).await; + assert_eq!( + status, + reqwest::StatusCode::FORBIDDEN, + "viewer must not start another user's job early (got {status}): {body}" + ); + let (status, body) = post(&authed_base, &run_now, Some("SECRET_TOKEN_2")).await; + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "a running job has no future start to move (got {status}): {body}" + ); + sqlx::query( + "UPDATE v2_job_queue SET running = false, scheduled_for = now() + interval '1 hour' + WHERE id = $1::uuid", + ) + .bind(RUNNING_JOB) + .execute(&db) + .await?; + let (status, body) = post(&authed_base, &run_now, Some("SECRET_TOKEN_2")).await; + assert!( + status.is_success(), + "owner must start their deferred job now (got {status}): {body}" + ); + let due: bool = + sqlx::query_scalar("SELECT scheduled_for <= now() FROM v2_job_queue WHERE id = $1::uuid") + .bind(RUNNING_JOB) + .fetch_one(&db) + .await?; + assert!(due, "run_now must make the job due immediately"); + + // A schedule tick that is not due yet is refused: the schedule would queue that same + // tick again on completion and run twice. + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, schedule, script_path, permissioned_as) + VALUES ('test-workspace', 'u/test-user-2/daily', 'test-user-2', '0 0 3 * * *', + 'u/test-user-2/running_secret', 'u/test-user-2')", + ) + .execute(&db) + .await?; + sqlx::query( + "UPDATE v2_job SET trigger_kind = 'schedule', trigger = 'u/test-user-2/daily' + WHERE id = $1::uuid", + ) + .bind(RUNNING_JOB) + .execute(&db) + .await?; + sqlx::query( + "UPDATE v2_job_queue SET scheduled_for = + (date_trunc('day', now() AT TIME ZONE 'UTC') + interval '1 day 3 hours') AT TIME ZONE 'UTC' + WHERE id = $1::uuid", + ) + .bind(RUNNING_JOB) + .execute(&db) + .await?; + let (status, body) = post(&authed_base, &run_now, Some("SECRET_TOKEN_2")).await; + assert_eq!( + status, + reqwest::StatusCode::BAD_REQUEST, + "an upcoming schedule tick must not be started early (got {status}): {body}" + ); + // A tick a concurrency limit pushed past its time is off the cron occurrences and + // may still be started early. + sqlx::query( + "UPDATE v2_job_queue SET scheduled_for = now() + interval '1 hour 37 seconds 123 milliseconds' + WHERE id = $1::uuid", + ) + .bind(RUNNING_JOB) + .execute(&db) + .await?; + let (status, body) = post(&authed_base, &run_now, Some("SECRET_TOKEN_2")).await; + assert!( + status.is_success(), + "a deferred schedule tick must still start now (got {status}): {body}" + ); + sqlx::query("UPDATE v2_job SET trigger_kind = NULL, trigger = NULL WHERE id = $1::uuid") + .bind(RUNNING_JOB) + .execute(&db) + .await?; + // The owner still cancels their own job (no over-blocking). Keep this last: it // takes RUNNING_JOB out of the queue. let (status, body) = post( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4346dfcd0a..21ca421af3 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -16310,6 +16310,29 @@ paths: items: type: string + /w/{workspace}/jobs/queue/run_now/{id}: + post: + summary: start a queued job scheduled for later now + description: | + Moves the start of a queued job that is waiting for a future time (scheduled, debounced, + re-scheduled by a concurrency limit, or a flow step waiting on a sleep or a retry delay) + to now, keeping the same job id. A job with a + concurrency limit is still checked against its limit when it is pulled. A schedule's + upcoming tick is refused, since the schedule would queue that tick again. + operationId: runQueuedJobNow + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: job id + content: + text/plain: + schema: + type: string + /w/{workspace}/jobs/queue/cancel_selection: post: summary: cancel jobs based on the given uuids @@ -31177,6 +31200,7 @@ components: - "jobs" - "jobs.cancel" - "jobs.force_cancel" + - "jobs.run_now" - "jobs.disapproval" - "jobs.delete" - "account.delete" diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 052c9df4bb..b9e71271d4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -116,7 +116,9 @@ use windmill_common::{ query_builders, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, - utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath}, + utils::{ + not_found_if_none, now_from_db, paginate, require_admin, Pagination, ScheduleType, StripPath, + }, }; use windmill_common::{ @@ -304,6 +306,7 @@ pub fn workspaced_service() -> Router { .route("/queue/position/{timestamp}", get(get_queue_position)) .route("/queue/scheduled_for/{id}", get(get_scheduled_for)) .route("/queue/cancel_selection", post(cancel_selection)) + .route("/queue/run_now/{id}", post(run_queued_job_now)) .route("/completed/count", get(count_completed_jobs)) .route("/completed/count_jobs", get(count_completed_jobs_detail)) .route( @@ -835,6 +838,113 @@ async fn force_cancel( } } +/// Moves a queued job's `scheduled_for` to now so the next pull picks it up, keeping its id. +/// A job with a concurrency limit still goes through the usual check at pull time and is +/// re-scheduled again if its key is still saturated. +async fn run_queued_job_now( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::Result { + require_job_update_read_access(&db, &user_db, &authed, &w_id, &id, None).await?; + refuse_upcoming_schedule_tick(&db, &w_id, id).await?; + + let mut tx = db.begin().await?; + let previous = sqlx::query_scalar!( + "UPDATE v2_job_queue q SET scheduled_for = now() + FROM (SELECT id, scheduled_for FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 FOR UPDATE) prev + WHERE q.id = prev.id AND NOT q.running AND q.suspend = 0 AND q.canceled_by IS NULL + AND q.scheduled_for > now() + RETURNING prev.scheduled_for", + id, + w_id, + ) + .fetch_optional(&mut *tx) + .await?; + + let Some(previous) = previous else { + tx.commit().await?; + return Err( + if sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2)", + id, + w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false) + { + Error::BadRequest(format!( + "job {id} is not waiting for a future start: it is running, suspended, \ + canceled or already due" + )) + } else { + Error::NotFound(format!("queued job id {id} does not exist")) + }, + ); + }; + + audit_log( + &mut *tx, + &authed, + "jobs.run_now", + ActionKind::Update, + &w_id, + Some(&id.to_string()), + Some([("previous_scheduled_for", previous.to_rfc3339().as_str())].into()), + ) + .await?; + tx.commit().await?; + + windmill_queue::append_logs( + &id, + &w_id, + format!( + "\nStart moved from {previous} to now by {}\n", + authed.display_username() + ), + &db.clone().into(), + ) + .await; + + Ok(id.to_string()) +} + +/// A schedule queues its next tick when the current one completes, computed from the +/// completion time. Starting a tick that is not yet due would therefore queue that same +/// tick again and run the schedule twice. A tick pushed past its time by a concurrency +/// limit no longer sits on a cron occurrence, so it may still be started early. +async fn refuse_upcoming_schedule_tick(db: &DB, w_id: &str, id: Uuid) -> error::Result<()> { + let Some(tick) = sqlx::query!( + "SELECT q.scheduled_for, s.path, s.schedule, s.cron_version, s.timezone + FROM v2_job j + JOIN v2_job_queue q USING (id) + JOIN schedule s ON s.workspace_id = j.workspace_id AND s.path = j.trigger + WHERE j.id = $1 AND j.workspace_id = $2 AND j.trigger_kind = 'schedule' + AND j.parent_job IS NULL AND q.scheduled_for > now()", + id, + w_id, + ) + .fetch_optional(db) + .await? + else { + return Ok(()); + }; + let sched = ScheduleType::from_str(&tick.schedule, tick.cron_version.as_deref(), false)?; + let tz = + chrono_tz::Tz::from_str(&tick.timezone).map_err(|e| Error::BadRequest(e.to_string()))?; + let just_before = (tick.scheduled_for - chrono::Duration::milliseconds(1)).with_timezone(&tz); + if sched.find_next(&just_before)? == tick.scheduled_for { + return Err(Error::BadRequest(format!( + "job {id} is the upcoming tick of schedule {}; run the schedule's script or flow \ + directly instead", + tick.path + ))); + } + Ok(()) +} + #[derive(Serialize)] struct QueuePosition { position: Option, diff --git a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte index 96d341dc72..4090a39dbb 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte @@ -149,6 +149,7 @@ JOBS: 'jobs', JOBS_CANCEL: 'jobs.cancel', JOBS_FORCE_CANCEL: 'jobs.force_cancel', + JOBS_RUN_NOW: 'jobs.run_now', JOBS_DISAPPROVAL: 'jobs.disapproval', JOBS_DELETE: 'jobs.delete', JOBS_SHARE_PUBLICLY: 'jobs.share_publicly', diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 932a76ae94..5fc7c1209d 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -40,8 +40,10 @@ EllipsisVertical, Share2, Globe, - Users + Users, + Play } from 'lucide-svelte' + import { forLater } from '$lib/forLater' import { isJobResolvable } from '$lib/utils' import { @@ -280,6 +282,27 @@ } } + // A schedule's upcoming tick sits on a whole second and the backend refuses it; a tick + // deferred by a concurrency limit lands on a sub-second instant and can still start now. + let canRunNow = $derived( + job?.type === 'QueuedJob' && + !job.running && + !job.suspend && + !!job.scheduled_for && + forLater(job.scheduled_for) && + !(job.schedule_path && new Date(job.scheduled_for).getMilliseconds() === 0) + ) + + async function runJobNow(id: string) { + try { + await JobService.runQueuedJobNow({ workspace: $workspaceStore!, id }) + sendUserToast(`job ${id} will start as soon as a worker is available`) + getJob() + } catch (err) { + sendUserToast(`could not start job now: ${err?.body ?? err}`, true) + } + } + // Initialize view tab to logs since result is now outside tabs function initView(): void { // Result is now displayed outside tabs, so always default to logs @@ -886,6 +909,17 @@ Current runs {/if} + {#if canRunNow} + + {/if} {#if job && job?.type != 'CompletedJob' && (!job?.schedule_path || job?.['running'] == true)} {#if !forceCancel} + {#if !!actingUser?.is_admin || !!$superadmin} + + {/if}
    {/if} - {#each treeRoots as root (root.datatable ?? '')} - {@const dtOpen = isExpanded(root.datatable)} - {#if root.datatable !== undefined} - {@const hasMenu = !multiSelectMode && onDatatableAction !== undefined} - {@const roleInfo = roleOf(root.datatable)} - - {/if} - {#if dtOpen} - {#if root.error} -

    {root.error}

    {/if} - {#each root.schemas as sc (sc.schemaKey)} - {@const schemaOpen = isExpanded(root.datatable, sc.schemaKey)} - {@const indent = root.datatable !== undefined ? 'pl-7' : 'pl-3'} - {#if dbSupportsSchemas} + {#each treeRoots as root (root.datatable ?? '')} + {@const dtOpen = isExpanded(root.datatable)} + {#if root.datatable !== undefined} + {@const hasMenu = !multiSelectMode && onDatatableAction !== undefined} + {@const roleInfo = roleOf(root.datatable)} {/if} - - - {#if schemaOpen || !dbSupportsSchemas} - {@const tableIndent = dbSupportsSchemas - ? root.datatable !== undefined - ? 'pl-11' - : 'pl-7' - : root.datatable !== undefined - ? 'pl-7' - : 'pl-3'} - {#each sc.tables as tableKey (tableKey)} - {@const entry = { - datatable: root.datatable, - schema: sc.schemaKey, - table: tableKey - }} - {@const hasMenu = !multiSelectMode} - {@const isSelected = - root.datatable === currentDatatable && - selected.schemaKey === sc.schemaKey && - selected.tableKey === tableKey} + {#if dtOpen} + {#if root.error} +

    {root.error}

    + {/if} + {#each root.schemas as sc (sc.schemaKey)} + {@const schemaOpen = isExpanded(root.datatable, sc.schemaKey)} + {@const indent = root.datatable !== undefined ? 'pl-7' : 'pl-3'} + {#if dbSupportsSchemas} - {/each} - {#if canCreateTableIn(root.datatable, sc.schemaKey)} - {/if} + + + {#if schemaOpen || !dbSupportsSchemas} + {@const tableIndent = dbSupportsSchemas + ? root.datatable !== undefined + ? 'pl-11' + : 'pl-7' + : root.datatable !== undefined + ? 'pl-7' + : 'pl-3'} + {#each sc.tables as tableKey (tableKey)} + {@const entry = { + datatable: root.datatable, + schema: sc.schemaKey, + table: tableKey + }} + {@const hasMenu = !multiSelectMode} + {@const isSelected = + root.datatable === currentDatatable && + selected.schemaKey === sc.schemaKey && + selected.tableKey === tableKey} + + {/each} + {#if canCreateTableIn(root.datatable, sc.schemaKey)} + + {/if} + {/if} + + {/each} + {#if dbSupportsSchemas && search.trim() === '' && canCreateSchemaIn(root.datatable)} + {/if} -     - {/each} - {#if dbSupportsSchemas && search.trim() === '' && canCreateSchemaIn(root.datatable)} - - {/if} - {/if} - {/each} -
    - - - {#if mainPane} - {@render mainPane()} - {:else if viewMode === 'diagram'} - { - onViewMode?.('data') - selectTable(currentDatatable, schema, table) - }} - /> - {:else if tableKey && colDefs?.[tableKey]?.length} - {@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey, whereClause })} - (rowFilter = undefined)} - bind:this={_dbTable} - /> - {:else if databaseIsEmpty} -
    - Database is empty - {#if onImport} -
    - - {#if !!actingUser?.is_admin || !!$superadmin} - {/if} -
    + {/each} +
    +     + + + {#if mainPane} + {@render mainPane()} + {:else if tabs} + + {#each tabs.tabs as tab (tab.id)} + {#if mountedTabs.has(tab.id)} +
    + {#if tab.kind === 'data'} + {@render dataView( + tab.id, + tab.schema ?? ownSelected.schemaKey, + tab.table, + tab.id === tabs.currentData?.id + )} + {:else if tab.kind === 'diagram'} + {@render diagramView(tab.tables ?? [])} + {:else} + {@render sqlTab?.(tab)} + {/if} +
    + {/if} + {/each} + {:else if viewMode === 'diagram'} + {@render diagramView(diagramTables)} + {:else} + {@render dataView('', selected.schemaKey, selected.tableKey, true)} {/if} -
    - {/if} - - + + + {/if} +
    (aclDrawer = undefined)}> diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index b88b1d1510..8894043688 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -1,4 +1,8 @@ tabs?.update(tab.id, { code })} + onSchemaChange={() => refresh()} + schema={shownSchema} + /> + {/snippet} - - -
    { - // Only proceed if the click is directly on this div and not on the child elements - if (e.target === e.currentTarget) { - replResultData = undefined - } - }} - > - {#if replResultData} - {#key replResultData} - - {/key} - {/if} -
    + {#if !tabbed} + + +
    { + // Only proceed if the click is directly on this div and not on the child elements + if (e.target === e.currentTarget) { + replResultData = undefined + } + }} + > + {#if replResultData} + {#key replResultData} + + {/key} + {/if} +
    + {/if} + dbTableOpsFactory={({ colDefs, tableKey }) => dbTableOpsWithPreviewScripts({ colDefs, tableKey, - whereClause, input: _input, workspace: ws, tag: workerTag @@ -373,9 +429,10 @@ bind:this={_dbManager} {disabledTables} features={getDbFeatures(input)} + tableLayoutKey={(schema, table) => dbTableLayoutStorageKey(ws, _input, schema, table)} />     - {#if showRepl} + {#if showRepl && !tabbed} refresh()} - placeholderTableName={sortArray( - Object.keys( - shownSchema.schema[ - 'public' in shownSchema.schema - ? 'public' - : 'dbo' in shownSchema.schema - ? 'dbo' - : Object.keys(shownSchema.schema ?? {})?.[0] - ] ?? {} - ) - )?.[0]} + {placeholderTableName} + schema={shownSchema} /> {/if} diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 2d4060a59c..bd016544b0 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -8,21 +8,25 @@ import Select from './select/Select.svelte' import { ArrowLeft, + Code, Copy, Download, - Expand, - Minimize, Network, + Plus, RefreshCcw, Table2, - Upload + Tag, + Upload, + X } from 'lucide-svelte' import DBManagerContent from './DBManagerContent.svelte' - import type { DbManagerViewMode, PendingRowAction } from './DBManager.svelte' - import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' - import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' + import DropdownV2 from './DropdownV2.svelte' + import { flip } from 'svelte/animate' + import type { PendingRowAction } from './DBManager.svelte' + import type { DbManagerTab, DbManagerTabKind } from './dbManagerTabs.svelte' import { logFeatureUsage } from '$lib/utils/featureUsage' import { getDbType } from './dbOps' + import { renderDbQuotedIdentifier } from './apps/components/display/dbtable/utils' import DataTableMigrationsButton from './workspaceSettings/DataTableMigrationsButton.svelte' import DataTablePermissionsButton from './workspaceSettings/DataTablePermissionsButton.svelte' import { resource } from 'runed' @@ -176,12 +180,8 @@ dbManagerContent?.clearReplResult() } - let windowWidth = $state(window.innerWidth) - let expand = $state(false) - $effect(() => { if (!open) { - expand = false uriState.closeDrawer() // An action asked for on one data table must not be waiting when the // drawer is next opened on another database — or on no data table at @@ -200,20 +200,55 @@ let hasReplResult = $state(false) - // Which view the manager shows. Held here, beside the control that switches it - // and outside the key that remounts the manager, so picking another data table - // or role stays on the view the user was reading. - let requestedViewMode = $state('data') - // Only PostgreSQL has a diagram; the manager clamps the mode itself, and this - // keeps the control off the header for a database that cannot show one. + // The tabs start where the right pane does, so a narrow tree pane would run them into the + // title: below the room the title needs, in pixels, it goes. + const TITLE_ROOM_PX = 200 + let showTitle = $derived.by(() => { + const left = dbManagerContent?.dbManager()?.mainPaneLeft() + return !dbManagerContent?.tabsModel() || !left || left >= TITLE_ROOM_PX + }) + + // Only PostgreSQL has a diagram. let diagramSupported = $derived( !!uriState.effectiveInput && getDbType(uriState.effectiveInput) === 'postgresql' ) + const TAB_KINDS: { kind: DbManagerTabKind; label: string; icon: typeof Table2 }[] = [ + { kind: 'data', label: 'Data', icon: Table2 }, + { kind: 'diagram', label: 'Diagram', icon: Network }, + { kind: 'sql', label: 'SQL Editor', icon: Code } + ] + /** A plain mouse wheel scrolls vertically, which the tab strip cannot: turn it sideways. + * Registered by hand because Svelte's `onwheel` is passive and cannot prevent the default. */ + function wheelScrollsSideways(el: HTMLElement) { + const onWheel = (e: WheelEvent) => { + if (e.deltaX || !e.deltaY || el.scrollWidth <= el.clientWidth) return + el.scrollLeft += e.deltaY + e.preventDefault() + } + el.addEventListener('wheel', onWheel, { passive: false }) + return () => el.removeEventListener('wheel', onWheel) + } + + function quotedTableName(schema: string | undefined, table: string): string { + const input = uriState.effectiveInput + if (!input) return table + const q = (name: string) => renderDbQuotedIdentifier(name, getDbType(input)) + return schema ? `${q(schema)}.${q(table)}` : q(table) + } + + let draggedTabId: string | undefined = $state() + let dropTarget: { id: string; side: 'before' | 'after' } | undefined = $state() + function tabLabel(tab: DbManagerTab): string { + if (tab.kind === 'data') return tab.table ?? 'Data' + return TAB_KINDS.find((k) => k.kind === tab.kind)!.label + } + // Export/Import state let exportDrawerOpen = $state(false) let exportResult = $state('') let importDrawerOpen = $state(false) + let workerTagOpen = $state(false) let importLoading = $state(false) let importSource = $state(undefined) /** Which database an import writes into; set when driven from a tree row. */ @@ -318,17 +353,9 @@ } - - - + { if (hasReplResult) { dbManagerContent?.clearReplResult() @@ -338,14 +365,16 @@ }} CloseIcon={hasReplResult ? ArrowLeft : undefined} noPadding + overflow_y={false} + fullScreen={false} + titleClass="text-sm text-primary" id="db-manager-drawer" > {#if contentInput && ws && roleSettled} {#key `${selectedDatatable}~${selectedRole ?? ''}`} (requestedViewMode = mode)} + tabbed input={contentInput} workspace={uriState.workspace} datatableTree={uriState.isDatatableInput ? datatables.current : undefined} @@ -357,9 +386,7 @@ uriState.selectedRole = role }} bind:pendingAction - canManageDatatable={!!($superadmin || $userStore?.is_admin) && - !!$enterpriseLicense && - !isCloudHosted()} + canManageDatatable={!!($superadmin || $userStore?.is_admin)} onDatatableAction={runDatatableAction} bind:workerTag={() => workerTag.tag, (v) => (workerTag.tag = v)} bind:hasReplResult @@ -375,38 +402,167 @@ > {/key} {/if} + {#snippet titleExtra()} + {@const mainPaneLeft = dbManagerContent?.dbManager()?.mainPaneLeft()} + {@const tabs = dbManagerContent?.tabsModel()} + + {#if tabs && mainPaneLeft} + +
    + {#each tabs?.tabs ?? [] as tab (tab.id)} + {@const active = tab.id === tabs?.activeId} + {@const Icon = TAB_KINDS.find((k) => k.kind === tab.kind)!.icon} + {@const dropSide = dropTarget?.id === tab.id ? dropTarget.side : undefined} + + +
    { + draggedTabId = tab.id + e.dataTransfer?.setData('text/plain', tab.id) + if (e.dataTransfer) e.dataTransfer.effectAllowed = 'move' + }} + ondragover={(e) => { + if (!draggedTabId) return + e.preventDefault() + if (draggedTabId === tab.id) { + dropTarget = undefined + return + } + const rect = e.currentTarget.getBoundingClientRect() + const side = e.clientX < rect.left + rect.width / 2 ? 'before' : 'after' + if (dropTarget?.id !== tab.id || dropTarget.side !== side) + dropTarget = { id: tab.id, side } + }} + ondrop={(e) => { + e.preventDefault() + if (draggedTabId && dropTarget) + tabs?.move(draggedTabId, dropTarget.id, dropTarget.side) + draggedTabId = undefined + dropTarget = undefined + }} + ondragend={() => { + draggedTabId = undefined + dropTarget = undefined + }} + {@attach (el) => { + if (active) el.scrollIntoView({ block: 'nearest', inline: 'nearest' }) + }} + > + {#if dropSide} +
    + {/if} + + +
    + {/each} + k.kind !== 'diagram' || diagramSupported).map((k) => ({ + displayName: k.label, + icon: k.icon, + action: () => { + // A query opened from a data tab starts on that tab's table. + const from = tabs?.active.kind === 'data' ? tabs.active : undefined + tabs?.add( + k.kind, + k.kind === 'sql' && from?.table + ? { + code: `SELECT * FROM ${quotedTableName(from.schema, from.table)}` + } + : {} + ) + logFeatureUsage('db_manager', 'view_mode', { key: k.kind }) + } + }))} + btnId="db-manager-new-tab" + > + {#snippet buttonReplacement()} +
    + +
    + {/snippet} +     +
    + {/if} + {/snippet} {#snippet actions()} - - {#if enableImportExport && !uriState.isDatatableInput} - - - {/if} - {#if diagramSupported} - logFeatureUsage('db_manager', 'view_mode', { key: v })} - > - {#snippet children({ item })} - - - {/snippet} - - {/if} {#if uriState.effectiveInput && ws} - workerTag.tag, (v) => (workerTag.tag = v)} - input={uriState.effectiveInput} - workspace={ws} - /> + {@const input = uriState.effectiveInput} + +
    + [ + { + displayName: workerTag.tag ? `Worker tag: ${workerTag.tag}` : 'Worker tag', + icon: Tag, + action: () => (workerTagOpen = true) + }, + // A data table exports and imports from its row menu in the tree; a + // plain database has no tree row to hold them. + ...(enableImportExport && !uriState.isDatatableInput + ? [ + { + displayName: 'Export database', + icon: Download, + action: () => handleExportSchema() + }, + { + displayName: 'Import database', + icon: Upload, + action: () => ((importTarget = undefined), (importDrawerOpen = true)) + } + ] + : []) + ]} + btnId="db-manager-database-actions" + /> + workerTag.tag, (v) => (workerTag.tag = v)} + bind:open={workerTagOpen} + hideTrigger + {input} + workspace={ws} + /> +
    {/if} - +
    + {/snippet} + +
    + {/if} + {@render hiddenColumnsNote()} +{/snippet} + +{#snippet addColumnCell()} + {#if canAddColumn} +
    + {/if} + {#if hidden.length} +
    + {/if} +{/snippet} + + { + if (editing) cancelEdit() + closeFormatOnOutsideClick(e) + }} + onkeydown={(e) => { + if (formatting && e.key === 'Escape') formatting = undefined + }} +/>
    -
    -
    - + +
    + {#if totalRows > 0 && endRow !== 0} + + {Math.min(firstVisible + 1, totalRows)}–{Math.min(lastVisible, totalRows)} + {#if rowCount !== undefined || endRow !== undefined} + of {endRow ?? rowCount} rows + {/if} + + {/if} + + {/snippet} + {#snippet content()} +
    + +
    + {#each columns as col (col.field)} +
    + + {#if formats[col.field]} + + {/if} + openFormat(col.field) + } + ]} + size="2xs" + class="opacity-0 group-hover:opacity-100 focus-within:opacity-100" + /> +
    + {/each} +
    + {#if onNewColumn} +
    + +
    + {/if} +
    + {/snippet} + +
    + {#if dbTableOps.onInsert} + { + if (!$operatingWorkspace) return + dbTableOps.onInsert?.({ values }).then(() => { + refresh() + sendUserToast('Row inserted') + }) + }} + /> {/if}
    - {#if dbTableOps.onInsert} - { - if (!$operatingWorkspace) return - dbTableOps.onInsert?.({ values }).then((result) => { - refresh?.() - sendUserToast('Row inserted') - }) - }} - /> - {/if}
    -
    - + +
    { - if ((e.ctrlKey || e.metaKey) && e.key === 'c') { - const selectedCell = api?.getFocusedCell() - if (selectedCell) { - const rowIndex = selectedCell.rowIndex - const colId = selectedCell.column?.getId() - const rowNode = api?.getDisplayedRowAtIndex(rowIndex) - const selectedValue = rowNode?.data?.[colId] - navigator.clipboard.writeText(selectedValue) - sendUserToast('Copied cell value to clipboard', false) + bind:this={scrollEl} + bind:clientHeight={viewportHeight} + bind:clientWidth={viewportWidth} + bind:offsetHeight={gridOuterHeight} + bind:offsetWidth={gridOuterWidth} + class={twMerge( + 'h-full w-full overflow-x-auto overflow-y-scroll relative outline-none text-xs text-primary', + // Wider than the app's default so it is easy to grab. + '[&::-webkit-scrollbar]:w-2.5 [&::-webkit-scrollbar]:h-2.5 [&::-webkit-scrollbar-thumb]:rounded-full' + )} + tabindex="0" + role="grid" + aria-rowcount={totalRows} + data-testid="db-table-grid" + onscroll={(e) => (scrollTop = e.currentTarget.scrollTop)} + onkeydown={onGridKeyDown} + onclick={(e) => (selected = cellFromEvent(e))} + ondblclick={(e) => { + const cell = cellFromEvent(e) + if (cell) startEdit(cell.row, cell.column) + }} + > +
    + +
    + {#each displayColumns as col (col.field)} + {@const sorted = sort?.column === col.field ? sort : undefined} + {@const pin = pinned[col.field]} + {#if col.field === firstRightPinned} + {@render addColumnHeader()} +
    + {/if} +
    + + +
    startResize(e, col.field)} + ondblclick={(e) => { + e.stopPropagation() + autoSize([col.field]) + }} + >
    +
    + {/each} + {#if !firstRightPinned} + {@render addColumnHeader()} +
    + {/if} +
    + + {#if endRow === 0} +
    + {hasFilters ? 'No rows match the filters' : 'No rows'} +
    + {/if} + +
    + {#each renderedIndices as i (i)} + {@const data = rowAt(i)} +
    + {#each displayColumns as col (col.field)} + {@const isSelected = selected?.row === i && selected.column === col.field} + {@const pin = pinned[col.field]} + {@const style = + data && data !== 'loading' + ? cellStyle(data[col.field], col.datatype, formats[col.field]) + : undefined} + {#if col.field === firstRightPinned} + {@render addColumnCell()} +
    + {/if} +
    + {#if pin} + +
    + {/if} + {#if data === 'loading'} +
    + {:else if data} + {@const value = data[col.field]} + {@const format = formats[col.field]} + {@const shown = formatValue(value, format) ?? formatCell(value)} + {#if value === null || value === undefined} + NULL + {:else if fkByColumn[col.field] && hasForeignKeyValue(value)} + {@const fk = fkByColumn[col.field]} + + {:else} + {shown} + {/if} + {/if} +
    + {/each} + {#if !firstRightPinned} + {@render addColumnCell()} +
    + {/if} +
    + {/each} +
    +
    +
    +     + + {#if gridOuterWidth - viewportWidth > 0} +
    + {/if} + {#if gridOuterHeight - viewportHeight > 0} +
    + {/if} +
    +
    + + { + if (editing) { + const el = cellElement(editing.row, editing.column) + if (el) editorRect = el.getBoundingClientRect() + } + return editorRect + }} +> + +
    e.stopPropagation()}> + {#if editing} + { + if (e.key === 'Enter') { + e.preventDefault() + commitEdit() + } else if (e.key === 'Escape') { + e.preventDefault() + e.stopPropagation() + cancelEdit() } } }} - >
    -
    - -
    -
    - - {#snippet text()} - Download - {/snippet} -
    - {#if rowCount} - {firstRow}{'->'}{lastRow + 1} of {rowCount} rows - {:else} - {firstRow}{'->'}{lastRow + 1} - {/if} -
    + /> +
    Enter to save, Esc to cancel
    + {/if}
    -
    + + + { + if (formatting) { + const header = scrollEl?.querySelector(`[data-header-column="${CSS.escape(formatting)}"]`) + // A hidden column has no header: its format opens from the columns button. + const el = header ?? columnsAnchor + if (el) formatRect = el.getBoundingClientRect() + } + return formatRect + }} +> +
    + {#if formatting} + {@const column = formatting} + setFormat(column, f)} + /> + {/if} +
    +     diff --git a/frontend/src/lib/components/DbColumnFormatEditor.svelte b/frontend/src/lib/components/DbColumnFormatEditor.svelte new file mode 100644 index 0000000000..36f98574a1 --- /dev/null +++ b/frontend/src/lib/components/DbColumnFormatEditor.svelte @@ -0,0 +1,362 @@ + + +{#snippet sample(style: { bg?: string; text?: string; bold?: boolean; italic?: boolean })} + + Aa + +{/snippet} + +{#snippet presets(onPick: (preset: (typeof RULE_PRESETS)[number]) => void)} +
    + {#each RULE_PRESETS as preset, i (i)} + + {/each} +
    +{/snippet} + +
    +
    +
    + Format {column} + Only changes how values show in this view. +
    + +
    + +
    + Unit +
    + ({ label: String(i), value: i })) + ]} + bind:value={ + () => format?.decimals ?? AUTO_DECIMALS, + (v) => update({ decimals: v === AUTO_DECIMALS ? undefined : v }) + } + /> + format?.notation ?? 'plain', + (v) => update({ notation: v === 'plain' ? undefined : (v as 'thousands' | 'compact') }) + } + > + {#snippet children({ item })} + + + + {/snippet} + +
    + + 1234567.891 shows as {formatValue(1234567.891, format) ?? '1234567.891'} + +
    + +
    + Alignment + format?.align ?? 'auto', + (v) => update({ align: v === 'auto' ? undefined : (v as 'left' | 'right') }) + } + > + {#snippet children({ item })} + + + + {/snippet} + +
    + +
    + Color rules + {#each rules as rule, i (i)} + +
    { + if (dragFrom === undefined) return + e.preventDefault() + const r = e.currentTarget.getBoundingClientRect() + dropAt = { index: i, side: e.clientY < r.top + r.height / 2 ? 'above' : 'below' } + }} + ondrop={(e) => { + e.preventDefault() + dropRule() + }} + > + {#if dropAt?.index === i && dragFrom !== undefined} +
    + {/if} + +
    { + dragFrom = i + const row = e.currentTarget.parentElement + if (row) e.dataTransfer?.setDragImage(row, 8, row.offsetHeight / 2) + }} + ondragend={() => { + dragFrom = undefined + dropAt = undefined + }} + > + +
    +
    + = 4, =paid' }} + bind:value={() => rule.condition, (v) => setRule(i, { condition: v })} + /> +
    + setRule(i, { bg: e.currentTarget.value })} + /> + setRule(i, { text: e.currentTarget.value })} + /> +
    + {/each} +
    + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + {@render presets((preset) => { + update({ rules: [...rules, { condition: '', ...preset }] }) + addRuleOpen = false + })} + {/snippet} + +
    + + Conditions use the search bar's syntax: text contains, =value is exact, numbers take >, + >=, <, <= and !=. An empty condition matches every cell. Every matching rule applies, + a later one overriding an earlier one. + +
    +
    diff --git a/frontend/src/lib/components/DbForeignKeyTooltip.svelte b/frontend/src/lib/components/DbForeignKeyTooltip.svelte deleted file mode 100644 index 526d6b2b69..0000000000 --- a/frontend/src/lib/components/DbForeignKeyTooltip.svelte +++ /dev/null @@ -1,23 +0,0 @@ - - -
    -
    - References {targetTable}.{targetColumn} -
    - -
    diff --git a/frontend/src/lib/components/DbSqlTab.svelte b/frontend/src/lib/components/DbSqlTab.svelte new file mode 100644 index 0000000000..a22f89c883 --- /dev/null +++ b/frontend/src/lib/components/DbSqlTab.svelte @@ -0,0 +1,63 @@ + + + + + (result = data)} + /> + + + {#if resultOps} + + {#key resultOps} + + {/key} + {:else} +
    + Run a query to see its result here +
    + {/if} +     +     diff --git a/frontend/src/lib/components/DbWorkerTagButton.svelte b/frontend/src/lib/components/DbWorkerTagButton.svelte index 3cbb60dc53..70db48b956 100644 --- a/frontend/src/lib/components/DbWorkerTagButton.svelte +++ b/frontend/src/lib/components/DbWorkerTagButton.svelte @@ -12,24 +12,40 @@ input: DbInput /** Workspace the custom tags are read from; defaults to the navigation one. */ workspace?: string + /** Render no button: the picker is opened through `open`, and pops up where this is placed. */ + hideTrigger?: boolean + open?: boolean } - let { tag = $bindable(), input, workspace = undefined }: Props = $props() + let { + tag = $bindable(), + input, + workspace = undefined, + hideTrigger = false, + open = $bindable() + }: Props = $props() let defaultTag = $derived(getDefaultDbTag(input)) - + open ?? false, (v) => (open = v)} + class={hideTrigger ? 'w-0 h-0 overflow-hidden' : undefined} + triggerAttrs={hideTrigger ? { tabindex: -1, 'aria-hidden': true } : undefined} +> {#snippet trigger()} - + {#if !hideTrigger} + + {/if} {/snippet} {#snippet content()}
    diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 6aabc30c2b..5ec3c62b5e 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -141,6 +141,9 @@ awareness?: any | undefined folding?: boolean args?: Record | undefined + /** Schema to complete SQL against when the caller already holds it; otherwise it is + * looked up from `args.database`. */ + sqlSchema?: DBSchema | undefined useWebsockets?: boolean small?: boolean scriptLang: Preview['language'] | 'bunnative' | 'tsx' | 'jsx' | 'json' | undefined @@ -191,6 +194,7 @@ awareness = undefined, folding = false, args = undefined, + sqlSchema = undefined, useWebsockets = true, small = false, scriptLang, @@ -274,7 +278,8 @@ const sessionScopedChatManager = getContext('aiChatManager') // let graphqlService: MonacoGraphQLAPI | undefined = undefined - let dbSchema: DBSchema | undefined = $state(undefined) + let fetchedDbSchema: DBSchema | undefined = $state(undefined) + let dbSchema: DBSchema | undefined = $derived(sqlSchema ?? fetchedDbSchema) let destroyed = false const uri = computeUri( @@ -777,8 +782,8 @@ async function updateSchema(newSchemaRes: string | undefined) { if (typeof newSchemaRes === 'string') { const resourcePath = newSchemaRes.replace('$res:', '') - dbSchema = $dbSchemas[resourcePath] - if (dbSchema === undefined) { + fetchedDbSchema = $dbSchemas[resourcePath] + if (fetchedDbSchema === undefined) { $dbSchemas[resourcePath] = await getDbSchemas( lang === 'graphql' ? 'graphql' : (scriptLang ?? ''), resourcePath, @@ -787,9 +792,9 @@ { customTag } ) } - dbSchema = $dbSchemas[resourcePath] + fetchedDbSchema = $dbSchemas[resourcePath] } else { - dbSchema = undefined + fetchedDbSchema = undefined } } @@ -824,6 +829,9 @@ sqlSchemaCompletor = languages.registerCompletionItemProvider('sql', { triggerCharacters: ['.', ' ', '('], provideCompletionItems: function (model, position) { + // Registered for the whole `sql` language: every other mounted SQL editor + // would otherwise add its own schema's suggestions to this one's. + if (model !== editor?.getModel()) return { suggestions: [] } const textUntilPosition = model.getValueInRange({ startLineNumber: 1, startColumn: 1, diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index 3a4833866a..30adb56c7c 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -339,6 +339,14 @@ hideDropdownOnFreeText?: boolean // Notified whenever the dropdown's effective visibility changes onDropdownVisibleChange?: (visible: boolean) => void + size?: 'md' | 'lg' + // On the right the icon gives way to the clear button once there is text; on the left + // it stays put. + searchIconPosition?: 'left' | 'right' + // Caps the suggestions dropdown, which otherwise matches the searchbar's width. + dropdownMaxWidth?: number + // Gap in px between the searchbar and the dropdown below it. + dropdownOffsetY?: number } type SchemaT = FilterSchemaRec // TODO: Generic @@ -351,7 +359,11 @@ autofocus, hideDropdownOnFreeText = false, onDropdownVisibleChange, - inputId + inputId, + size = 'md', + searchIconPosition = 'right', + dropdownMaxWidth, + dropdownOffsetY = 0 }: Props = $props() let _value = new DebouncedTempValue( @@ -702,7 +714,7 @@ class={twMerge( 'flex items-center rounded-md bg-surface-input overflow-clip', inputBorderClass({ error: errors.length > 0, forceFocus: open }), - ButtonType.UnifiedHeightClasses.md, + ButtonType.UnifiedHeightClasses[size], className )} onmousedown={(e) => { @@ -716,6 +728,9 @@ }} bind:this={inputElement} > + {#if searchIconPosition === 'left'} + + {/if} { @@ -752,7 +770,7 @@ /> {#if asText.val} (_value.current = {})} /> - {:else} + {:else if searchIconPosition === 'right'}
    @@ -762,7 +780,11 @@ inputElement?.getBoundingClientRect() ?? new DOMRect()} + getInputRect={() => { + const r = inputElement?.getBoundingClientRect() ?? new DOMRect() + const width = dropdownMaxWidth ? Math.min(r.width, dropdownMaxWidth) : r.width + return new DOMRect(r.x, r.y, width, r.height + dropdownOffsetY) + }} innerClass="!max-h-[25rem]" strictWidth > diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index d58ac65898..b5bdcb872a 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -1,5 +1,5 @@ - - - {#await import('$lib/components/Editor.svelte')} - - {:then Module} - - {/await} - - - - { - const data = e.detail as (typeof runHistory)[number] - editor?.setCode(data.code) - onData(data.result, data.code) - }} +
    + {#await import('$lib/components/Editor.svelte')} + + {:then Module} + code, (v) => ((code = v), onCodeChange?.(v))} + scriptLang="mysql" + sqlSchema={schema} + class="w-full h-full" + cmdEnterAction={run} /> - - + {/await} + + + {#if historyOpen} +
    +
    + Run history +
    +
    + { + const data = e.detail as (typeof runHistory)[number] + editor?.setCode(data.code) + onData(data.result, data.code) + }} + /> +
    +
    + {:else} +
    +
    + {/if} +
    {#if datatableName && ws} - import { Drawer, DrawerContent, Button } from '$lib/components/common' + import { Drawer, DrawerContent, Button, ButtonType } from '$lib/components/common' import { CornerDownLeft, Plus } from 'lucide-svelte' import InsertRow from './dbtable/InsertRow.svelte' import type { ColumnDef } from './dbtable/utils' @@ -9,9 +9,11 @@ columnDefs: ColumnDef[] dbType: DbType onInsert: (args: Record) => void | Promise + /** Sizes the trigger on the unified scale; without it the trigger keeps its legacy size. */ + unifiedSize?: ButtonType.UnifiedSize } - let { columnDefs, dbType, onInsert }: Props = $props() + let { columnDefs, dbType, onInsert, unifiedSize }: Props = $props() let args: Record = $state({}) let insertDrawer: Drawer | undefined = $state() @@ -35,7 +37,8 @@ + {#if unavailable} + + + + + {:else} + + {/if} {/if} From 1d119e6b25fd5c72776dfb49917387de4be065c1 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 25 Sep 2026 17:12:28 +0200 Subject: [PATCH 27/91] fix: correct the tool controls and name field of a nested AI agent (#11221) * fix: hide tool controls a nested AI agent cannot use Co-Authored-By: Claude Opus 5 * refactor: drop the tool navigation prop an agent tool now implies Co-Authored-By: Claude Opus 5 * fix: ask for a tool name on every kind of agent tool Co-Authored-By: Claude Opus 5 * fix: keep enabled_tools reachable on a linked nested agent tool Co-Authored-By: Claude Opus 5 * docs: name the tool kinds the header's name field actually renders Co-Authored-By: Claude Opus 5 * docs: state the tool-name rule without listing the kinds it covers Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../flows/common/FlowCardHeader.svelte | 29 +++++++++++-------- .../flows/content/AgentEditorHost.svelte | 1 - .../flows/content/AgentToolWrapper.svelte | 6 +--- .../flows/content/AiAgentStepInputs.svelte | 14 +++++++-- .../flows/content/FlowModuleComponent.svelte | 18 +++++++----- .../flows/map/InsertModuleInner.svelte | 1 + .../flows/pickers/TopLevelNode.svelte | 7 +++-- 7 files changed, 46 insertions(+), 30 deletions(-) diff --git a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte index 5776c88025..6d14463c6b 100644 --- a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte +++ b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte @@ -62,6 +62,10 @@ let toolNameError = $derived( isAgentTool ? getToolNameError(summary ?? '', undefined, siblingToolNames) : undefined ) + // An agent tool's summary is the function name the model is given, not free text, so the field + // asks for a name instead. That holds for every kind a tool can be, which is why the input + // below is one shared snippet rather than one per branch. + let summaryPlaceholder = $derived(isAgentTool ? 'Tool name' : 'Summary') const dispatch = createEventDispatcher() const customUi: FlowBuilderWhitelabelCustomUi | undefined = getContext('customUi') @@ -194,6 +198,14 @@ }) +{#snippet summaryInput()} + +{/snippet} +
    {/if} -
    - - {#if toolNameError && !isAgentTool} -

    {toolNameError}

    - {/if} +
    + {@render summaryInput()}
    {:else if flowModuleValue.type === 'flow'} flow - + {@render summaryInput()} {:else if flowModuleValue.type === 'aiagent'} AI Agent - + {@render summaryInput()} {/if}
    diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 4f69d3a1a1..f0d65068f0 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -599,7 +599,6 @@ parentModule={agentModule as FlowModule} {enableAi} staticOnly - noToolNavigation forceTestTab={readOnly ? undefined : { [tool.id]: true }} siblingToolNames={tools.filter((t) => t?.id !== tool?.id).map((t) => t?.summary ?? '')} /> diff --git a/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte b/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte index 6ccadc2ee5..69a4acf74e 100644 --- a/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte +++ b/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte @@ -18,8 +18,6 @@ /** See `FlowModuleComponent`: set when the tool belongs to a saved agent rather than to a * step of this flow. */ staticOnly?: boolean - /** See `FlowModuleComponent`: set where there is no graph to select a nested tool on. */ - noToolNavigation?: boolean } let { @@ -31,8 +29,7 @@ forceTestTab, highlightArg, siblingToolNames = undefined, - staticOnly = false, - noToolNavigation = false + staticOnly = false }: Props = $props() @@ -64,7 +61,6 @@ highlightArg={highlightArg?.[tool.id]} isAgentTool={true} {staticOnly} - {noToolNavigation} bind:toolDescription={tool.description} {siblingToolNames} /> diff --git a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte index bb0f260a21..e5529dce5c 100644 --- a/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte +++ b/frontend/src/lib/components/flows/content/AiAgentStepInputs.svelte @@ -117,6 +117,9 @@ /** A linked agent's memory, once its config has loaded: whether it keeps managed memory decides * which history inputs the step offers. */ linkedMemory?: { memory: unknown } | undefined + /** Set when this agent's config lives in a resource, so `tools` arrives with it rather than + * with the step. */ + agentLinked?: boolean } let { @@ -147,7 +150,8 @@ onAddTool = undefined, onDeleteTool = undefined, toolPickerPortal = undefined, - linkedMemory = undefined + linkedMemory = undefined, + agentLinked = false }: Props = $props() let ws = $derived(workspace ?? $operatingWorkspace) @@ -226,10 +230,16 @@ }) }) + // No editor adds tools to an agent used as a tool, so an empty roster is all an inline one will + // ever have. Not a linked one: its tools arrive with the resource, so empty here means "not + // loaded" as often as "none", and `enabled_tools` is the step's to set either way. + let toolsHidden = $derived(isAgentTool && !agentLinked && tools.length === 0) let scopedFields = $derived( AGENT_FIELDS.filter( (spec) => - agentFieldAppliesTo(spec, schemaProperties) && (!filter || filter.includes(spec.key)) + agentFieldAppliesTo(spec, schemaProperties) && + (!filter || filter.includes(spec.key)) && + !(toolsHidden && spec.group === 'tools') ) ) diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 0c87d6fb9e..a256a9998a 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -128,10 +128,6 @@ staticOnly?: boolean /** Lets the agent's Tools section add a tool through the graph's own insert path. */ flowModuleSchemaMap?: import('../map/FlowModuleSchemaMap.svelte').default - /** Drop the tool roster's drill-in. Selecting a tool means selecting its graph node, so a - * surface without a graph — the agent editor, which addresses one tool at a time — would - * offer a row whose click lands nowhere. */ - noToolNavigation?: boolean toolDescription?: string | undefined siblingToolNames?: string[] } @@ -152,7 +148,6 @@ isAgentTool = false, staticOnly = false, flowModuleSchemaMap = undefined, - noToolNavigation = false, toolDescription = $bindable(undefined), siblingToolNames = undefined }: Props = $props() @@ -206,6 +201,14 @@ let visibleSelected = $derived(selected === 'chat' && !canShowChatTab ? 'inputs' : selected) let runSettings: FlowRunSettings | undefined = $state() let agentLinked = $derived(flowModule.value.type === 'aiagent' && Boolean(flowModule.value.agent)) + // A tool row drills in by selecting that tool's graph node, and the graph draws tool nodes only + // for a step's own agent (`computeAIToolNodes`). An agent used as a tool has none, so its rows + // would offer a click that lands nowhere. + let onSelectToolInGraph = $derived( + isAgentTool + ? undefined + : (toolId: string) => selectionManager.selectId(toolId, { openPanel: true }) + ) let validCode = $state(true) let width = $state(1200) let testJob: Job | undefined = $state(undefined) @@ -1242,15 +1245,14 @@ workspace={opWs} visibilityKey={agentFieldsKey} linkedMemory={agentLinked ? linkedAgentMemory : undefined} + {agentLinked} tools={agentLinked ? getLinkedAgentTools( linkedToolsScope(opWs, $pathStore), linkedToolsModuleId ) : (flowModule.value.tools ?? [])} - onSelectTool={noToolNavigation - ? undefined - : (toolId) => selectionManager.selectId(toolId, { openPanel: true })} + onSelectTool={onSelectToolInGraph} onAddTool={flowModuleSchemaMap ? (detail) => flowModuleSchemaMap?.addToolToAgent(flowModule.id, detail) diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index 3c50e213f1..aea3d669ed 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -234,6 +234,7 @@ /> { dispatch('pickAiAgentTool') dispatch('close') diff --git a/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte b/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte index 8cd69e9e18..627b7c2174 100644 --- a/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte +++ b/frontend/src/lib/components/flows/pickers/TopLevelNode.svelte @@ -22,11 +22,14 @@ /** Highlight with the neutral hover surface instead of the accent, for transient * (hover/keyboard) selection rather than the persistent category selection. */ neutral?: boolean + /** Overrides the label's default chevron, for an entry that inserts directly here instead of + * opening a sub-menu. */ + chevron?: boolean onSelect: () => void onHover?: () => void } - let { label, selected, returnIcon, neutral = false, onSelect, onHover }: Props = $props() + let { label, selected, returnIcon, neutral = false, chevron, onSelect, onHover }: Props = $props() interface IconConfig { icon: ComponentType @@ -73,7 +76,7 @@ > {#if config} - {@render iconWithText(config.icon, config.showChevron, config.iconClass ?? '')} + {@render iconWithText(config.icon, chevron ?? config.showChevron, config.iconClass ?? '')} {/if} {#if returnIcon && selected} From 8caf41430123318a73b76e4c202da07aecaa3c1e Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 25 Sep 2026 11:12:47 -0400 Subject: [PATCH 28/91] fix(multiplayer): a malformed frame from one client no longer exits the server (#11353) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(multiplayer): don't drop client messages during cold-start token verification `wss.on('connection')` awaits `verifyToken()` before `setupWSConnection()` attaches the 'message' listener. On a cold process that await includes the first `/api/debug/jwks` fetch (~30ms on ECS). A y-websocket client sends sync step 1 the instant the socket opens, and `ws` drops messages emitted with no listener attached, so that step 1 was lost and never answered with step 2 — the client's provider never became `synced`. Buffer messages from the moment the connection is accepted and replay them, in order, once `setupWSConnection()` has installed its handlers. Rejected connections drop the buffer and close with the same 4401/4403 codes as before. Also prefetch the public key at startup when WINDMILL_BASE_URL is set. That is insurance, not the fix: a connection arriving before the prefetch resolves still relies on the buffer. Adds `npm test` in multiplayer/ (node:test, no docker or backend needed) with a fake JWKS endpoint that answers with a delay, which holds the cold window open and makes the race deterministic; wired into the existing test_extra CI job. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(multiplayer): cap what an unauthenticated peer can buffer pre-auth Review follow-up. The pre-auth buffer was unbounded: `ws` sets no `maxPayload` here and the JWKS fetch has no timeout, so a peer that never authenticates could stream frames into memory for as long as `verifyToken` was stalled. Cap it at 32 frames / 1 MiB — a real client only has sync step 1 and its first awareness update in flight there — and close 1009 past that, dropping what was buffered. A socket closed during verification (by the peer, or by that cap) is no longer handed to setupWSConnection: it would be added to `doc.conns` with a 'close' listener that can never fire. The startup prefetch's .catch was dead code — getPublicKey() logs its own failures and resolves to null rather than rejecting. Test helper: pin REQUIRE_SIGNED_MULTIPLAYER_REQUESTS and BASE_INTERNAL_URL so an ambient value cannot turn the rejection tests into false passes; bind the JWKS server on port 0 instead of a released probe port, and retry the spawned server on EADDRINUSE; destroy still-delayed JWKS responses on teardown, since server.close() waits for in-flight requests. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): gate the JWKS response instead of delaying it Review follow-up. The cold window was held open by a 1500 ms delay on the fake JWKS response, but that timer started when the startup prefetch reached the fake server, not when the client sent its first frame. A slow enough machine could load the key before the client connected, and the race test would then pass without ever exercising the buffer — a false pass. The fake JWKS server now parks every response until the test calls release(), so the server provably holds no key while the client is sending. The race test releases only after both frames are written to the socket, and asserts the server has not logged the key as loaded at that point; the flood test never releases until after the cap has closed the connection. What is left to wall-clock time is 250 ms for bytes already written to the socket to cross loopback into an otherwise idle server, rather than a window that had to cover process startup, connect and handshake. Also drops the prefetch precondition from the forged-token and flood tests so each test still maps to one behaviour. Suite runs in ~1.1s instead of ~5.3s. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(multiplayer): survive a malformed frame instead of exiting the process `setupWSConnection`'s message handler decoded whatever an authenticated peer put on the wire with no guard: `decoding.readVarUint`, `syncProtocol.readSyncMessage` and `awarenessProtocol.applyAwarenessUpdate` all throw on input they cannot parse, `ws` re-emits a listener's exception on the process, and server.mjs installs no `uncaughtException` handler. One bad frame from one client therefore killed the whole multiplayer server, taking every other document and every other client with it. Catch decode/apply failures, log the document, the client address and the error message (never the payload), and close only the offending connection with 1007 "invalid frame payload data". Frames that arrive once a connection is no longer OPEN are ignored, so the replay of the pre-auth buffer stops at the first refusal instead of applying the rest. docker/entrypoint-extra.sh made that outage permanent: on a service exit it logged a bare PID and then `wait`ed on the rest, so the container stayed up with a dead service and the health checks in front of it — which probe the LSP — saw nothing wrong. It now names the service that died, stops the others through the same shutdown path SIGTERM uses, and exits non-zero so the orchestrator replaces the container. The "no services enabled" branch still sleeps. Tests: multiplayer/test/malformed_frame.test.mjs covers four malformed payloads from an authenticated client and one replayed out of the pre-auth buffer, asserting the 1007 close, a live server process, an undisturbed bystander and a real edit still propagating. docker/test_entrypoint_extra.sh runs the real entrypoint in a container with stub services. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): prove the replayed malformed frame really is buffered pre-auth Assert the server has not yet logged the loaded key when the frame is written, and give it the same in-flight margin as the cold-start tests before releasing the JWKS response, so the frame provably goes through the replay path rather than landing on an already-authenticated connection. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(extra): run the entrypoint supervision tests in publish_extra The multiplayer unit tests already run there; the entrypoint test needs only docker and the checkout, so run it in the same job, before the image build. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(multiplayer): handle WebSocket protocol errors and bound the shutdown Two crash paths of the same class as the malformed-frame one, from review. `ws` fails a frame it cannot parse at the protocol level — an unmasked frame from a client, a reserved opcode, a bad RSV bit — inside its Receiver, before the application 'message' handler ever sees it, and `receiverOnError` ends with `websocket.emit('error', err)`. With no 'error' listener that is an unhandled EventEmitter error, so it exited the process just as a malformed payload did. (A raw socket error such as ECONNRESET does not: ws 8.21.3's `socketOnError` swallows those.) Add the listener on the accepted socket, before authentication so the pre-auth window is covered too, and one on the server. Log messages now go through `describeError`, which collapses whitespace and truncates, so nothing that reaches an error message can forge or flood a log line. `stop_services` ended in a bare `wait`. On the `docker stop` path dockerd provides the deadline; the "a service died" path signals itself, so a service that is wedged or slow to honour SIGTERM would hold the container open indefinitely — the state that path exists to prevent. Bound it: SIGTERM, wait SHUTDOWN_GRACE_SECS (10 by default), then SIGKILL the stragglers by name. Tests: multiplayer/test/socket_error.test.mjs (authenticated and pre-auth illegal frames, asserting a live process and continued service), a SIGTERM-ignoring stub scenario in docker/test_entrypoint_extra.sh, and that harness is now bounded throughout — `timeout -k` on foreground runs, a watchdog around the backgrounded ones, and an optional outer timeout on `docker run`. `--entrypoint bash` so the documented windmill-extra:test override runs the harness instead of the image's real entrypoint. The stubs publish a readiness marker and the dying one waits for them, removing a startup race in the harness. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(multiplayer): never log peer bytes, and keep a fatal server error fatal Two review findings on the previous commit, both mine to answer for. `describeError` collapsed whitespace, which is not enough. An error message is not always a fixed string: `applyAwarenessUpdate` runs `JSON.parse` on the peer's bytes and V8 quotes ~30 bytes of the offending input back verbatim, ESC included, so a peer could put terminal escapes and forged content into a log line. Strip everything outside printable ASCII instead, and say so where the comment previously claimed the messages were fixed strings. `wss.on('error')` was worse than the crash it replaced for one case: `ws` forwards the HTTP server's errors there, so a failed listen (EADDRINUSE) was logged and the process then exited 0 — a clean shutdown as far as anything upstream could tell. It now sets a non-zero exit code. Setting `process.exitCode` rather than calling `process.exit()` keeps the log line from being truncated. `openClient` in the test helpers now records the socket error it was already swallowing, so a failed connection reports its cause instead of surfacing as a bare `waitFor` timeout. Tests: a malformed awareness frame whose state is `x\x1b[2J OWNED THE LOG` added to the payload table, with every case now asserting exactly one refusal line and no control characters in it (1 fail before, 0 after, 3 runs); and a server that cannot listen must exit non-zero (1 fail before, 0 after, 3 runs). 14/14 on 5 consecutive runs. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): make the exit-status helper robust to spawn and stdio races Review follow-ups on runMultiplayerServerUntilExit. Wait for 'close', not 'exit': 'exit' fires when the child terminates, which can be before its stdio pipes are drained, and the caller reads the output. On the EADDRINUSE path the child writes one line and exits immediately after, which is exactly the shape that loses it. Listen for 'error' too. A child that fails to spawn emits neither 'exit' nor 'close', so the promise would never settle and the SIGKILL guard could not help. Report whether the guard fired, rather than leaving the caller to infer it from the exit signal: `signal` is null for every child exit on Windows, so a server that hung after the listen error would have looked like one that exited on its own. The test asserts on that flag instead. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): assert the refusal line itself, and name hasSyncType for what it takes Two review nits on the test helpers. The `doc="..."` assertion searched the whole server log, where CONNECT and DISCONNECT also name the document, so it would have passed even if the refusal stopped naming anything. Every assertion about the refusal is now made against the refusal line, which the test already isolates, and it also checks the peer is named. `hasKind` took a sync sub-type but was named as if it took any message kind, and the two families overlap numerically (`syncStep1 === messageSync === 0`), so a caller passing the wrong one got a silently wrong answer. No runtime check can tell aliased numbers apart, so the fix is the name: `hasSyncType`, with the overlap spelled out where the constants are declared. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): make the pre-auth tests prove which path they took The replay test could not establish that its frame went through the pre-auth buffer: a frame delivered after setup, into the live handler, produces the same close code and the same refusal line, so if the in-flight margin were ever missed the test would quietly become a duplicate of the main-loop cases rather than fail. server.mjs now logs REPLAY when, and only when, it replays a buffered pre-auth message — worth having on its own, since that path only runs when a client beat the JWKS fetch on a slow-starting instance — and the test asserts on it. Removing that log line turns the test red, which is the point. The socket-error pre-auth test gated on `jwks.requests >= 1`, which the startup warm-up already satisfies, so it proved nothing about the offender. What makes it the pre-auth case is that the JWKS response stays parked for the whole test; it now asserts the server never logged CONNECT, which is exact. `killedByTimeout` was set before the kill, so a child that exited on its own just before the timeout — with 'close' still pending on the stdio drain, the very window this helper waits for — would have been reported as killed. It now claims the rescue only when there was a live process to signal. 14/14 on eight consecutive runs. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): document the frame-recording contract in openClient ws hands every frame over as a Buffer under the default binaryType, text frames included, so recording them as Uint8Array is lossless for both. Worth stating: ws 7 delivered text frames as strings, where new Uint8Array(string) would have been a silent zero-fill, and the difference is not visible at the call site. Co-Authored-By: Claude Opus 5.5 (1M context) * test(multiplayer): pin the close code for an illegal frame, and trim comments to the 4-line rule Both socket-error tests waited for a close and never checked what it was, so an abrupt 1006 teardown would have passed while the comment beside the payload claimed 1002. `ws` sends 1002 for an unmasked frame in both the authenticated and pre-auth cases, confirmed over repeated runs; that is now a named constant asserted in each test, mirroring malformed_frame.test.mjs. Changing the expected value turns both red. The comments added by this branch also ran past the four lines AGENTS.md allows, and several justified the change to a reader rather than stating the invariant. Condensed to the invariant, at the site that would break it. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/publish_extra.yml | 5 + docker/entrypoint-extra.sh | 65 +++++- docker/test_entrypoint_extra.sh | 246 ++++++++++++++++++++++ multiplayer/server.mjs | 73 +++++-- multiplayer/test/cold_start.test.mjs | 65 ++---- multiplayer/test/helpers.mjs | 52 +++++ multiplayer/test/malformed_frame.test.mjs | 231 ++++++++++++++++++++ multiplayer/test/protocol.mjs | 94 +++++++++ multiplayer/test/socket_error.test.mjs | 128 +++++++++++ 9 files changed, 884 insertions(+), 75 deletions(-) create mode 100755 docker/test_entrypoint_extra.sh create mode 100644 multiplayer/test/malformed_frame.test.mjs create mode 100644 multiplayer/test/protocol.mjs create mode 100644 multiplayer/test/socket_error.test.mjs diff --git a/.github/workflows/publish_extra.yml b/.github/workflows/publish_extra.yml index 980eb1da55..863c850266 100644 --- a/.github/workflows/publish_extra.yml +++ b/.github/workflows/publish_extra.yml @@ -48,6 +48,11 @@ jobs: npm ci --omit=dev npm test + # Runs the real entrypoint-extra.sh with stub services, so the supervision + # behaviour is checked without building the image first. + - name: Run entrypoint supervision tests + run: bash docker/test_entrypoint_extra.sh + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/docker/entrypoint-extra.sh b/docker/entrypoint-extra.sh index b32ad2dd94..84b337a8a1 100644 --- a/docker/entrypoint-extra.sh +++ b/docker/entrypoint-extra.sh @@ -4,18 +4,51 @@ set -e # Windmill Extra Services Entrypoint # Starts LSP, Multiplayer, and Debugger services based on environment variables -# Track PIDs for cleanup +# Track PIDs for cleanup, with a parallel array of names so a dead service can be +# named in the logs rather than reported as a bare PID. PIDS=() +NAMES=() -cleanup() { +# How long a service gets to honour SIGTERM before it is killed outright. The +# exit path below signals itself, so nothing else bounds it there; a wedged +# service would otherwise hold the container open half-dead forever. +SHUTDOWN_GRACE_SECS="${SHUTDOWN_GRACE_SECS:-10}" + +stop_services() { echo "[entrypoint] Shutting down services..." for pid in "${PIDS[@]}"; do if kill -0 "$pid" 2>/dev/null; then kill "$pid" 2>/dev/null || true fi done + + local i running pid + for ((i = 0; i < SHUTDOWN_GRACE_SECS * 10; i++)); do + running=0 + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + running=1 + fi + done + if [ "$running" -eq 0 ]; then + break + fi + sleep 0.1 + done + + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo "[entrypoint] WARNING: PID $pid did not stop within ${SHUTDOWN_GRACE_SECS}s, killing it" >&2 + kill -9 "$pid" 2>/dev/null || true + fi + done + wait echo "[entrypoint] All services stopped" +} + +cleanup() { + stop_services exit 0 } @@ -116,6 +149,7 @@ if [ "${ENABLE_LSP:-true}" = "true" ]; then cd /pyls PORT=${LSP_PORT:-3001} python3 pyls_launcher.py & PIDS+=($!) + NAMES+=("LSP") echo "[entrypoint] LSP started (PID: ${PIDS[-1]})" fi @@ -125,6 +159,7 @@ if [ "${ENABLE_MULTIPLAYER:-true}" = "true" ]; then cd /multiplayer PORT=${MULTIPLAYER_PORT:-3002} HOST=${HOST:-0.0.0.0} node server.mjs & PIDS+=($!) + NAMES+=("Multiplayer") echo "[entrypoint] Multiplayer started (PID: ${PIDS[-1]})" fi @@ -144,6 +179,7 @@ if [ "${ENABLE_DEBUGGER:-true}" = "true" ]; then bun run dap_debug_service.ts $DEBUGGER_ARGS & PIDS+=($!) + NAMES+=("Debugger") echo "[entrypoint] Debugger started (PID: ${PIDS[-1]})" fi @@ -153,6 +189,7 @@ if [ "${ENABLE_GATEWAY:-true}" = "true" ]; then cd /multiplayer PORT=${GATEWAY_PORT:-3000} node gateway.mjs & PIDS+=($!) + NAMES+=("Gateway") echo "[entrypoint] Gateway started (PID: ${PIDS[-1]})" fi @@ -165,15 +202,27 @@ fi echo "[entrypoint] All enabled services started. Waiting..." -# Wait for any process to exit -wait -n "${PIDS[@]}" 2>/dev/null || true +# Wait for any process to exit. `|| status=$?` also keeps `set -e` from aborting +# here, which would skip the reporting and shutdown below. +status=0 +wait -n "${PIDS[@]}" 2>/dev/null || status=$? -# If one process exits, check which one and report +# Nothing here restarts a dead service, and the health checks in front of this +# container probe a single one, so a dead sibling is invisible and stays dead. +# Name it, stop the rest and exit non-zero, leaving the orchestrator to replace +# the container. for i in "${!PIDS[@]}"; do if ! kill -0 "${PIDS[$i]}" 2>/dev/null; then - echo "[entrypoint] Service (PID: ${PIDS[$i]}) has exited" + echo "[entrypoint] ERROR: ${NAMES[$i]} (PID: ${PIDS[$i]}) has exited" >&2 fi done -# Keep running and wait for remaining processes -wait +stop_services + +# Carry the dead service's own status where there is one, but never exit 0: that +# would read as a successful, deliberate shutdown. +if [ "$status" -eq 0 ]; then + status=1 +fi +echo "[entrypoint] Exiting with status $status so the container is restarted" >&2 +exit "$status" diff --git a/docker/test_entrypoint_extra.sh b/docker/test_entrypoint_extra.sh new file mode 100755 index 0000000000..e0b84da366 --- /dev/null +++ b/docker/test_entrypoint_extra.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# +# Supervision tests for docker/entrypoint-extra.sh. +# +# The real script is run unmodified inside a container, with the four services it +# starts replaced by stubs, so what is under test is the shipped file and the +# shipped bash. The stubs are recognised by the argv the entrypoint uses +# (pyls_launcher.py, server.mjs, dap_debug_service.ts, gateway.mjs), which is +# also what keeps the test honest: change how a service is started and the +# corresponding case here stops matching. +# +# Usage: +# bash docker/test_entrypoint_extra.sh +# +# The image defaults to the base of DockerfileExtra's chain (debian:trixie-slim, +# via windmill-ee-slim), so the bash built-ins behave as they do in the real +# image. Override with ENTRYPOINT_TEST_IMAGE to run it against another one, e.g. +# ENTRYPOINT_TEST_IMAGE=windmill-extra:test bash docker/test_entrypoint_extra.sh + +set -euo pipefail + +IMAGE="${ENTRYPOINT_TEST_IMAGE:-debian:trixie-slim}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENTRYPOINT="$SCRIPT_DIR/entrypoint-extra.sh" + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required to run this test" >&2 + exit 1 +fi + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +# --- the pieces that run inside the container ----------------------------- + +cat > "$WORKDIR/stub" <<'STUB' +#!/bin/bash +# Stands in for python3 / node / bun. Works out which service it was started as +# from its arguments, then either stays up until it is signalled (recording that +# it was stopped) or exits straight away, to play the crashed service. +name=unknown +for arg in "$@"; do + case "$arg" in + pyls_launcher.py) name=lsp ;; + server.mjs) name=multiplayer ;; + dap_debug_service.ts) name=debugger ;; + gateway.mjs) name=gateway ;; + esac +done +echo "stub[$name] started" + +if [ "$name" = "${STUB_IGNORE_TERM:-}" ]; then + # Plays a service that is wedged or slow to honour SIGTERM. + trap 'echo "stub[$name] ignoring SIGTERM"' TERM +else + trap 'echo "stub[$name] got SIGTERM"; touch "/out/$name.stopped"; exit 0' TERM +fi +touch "/out/$name.ready" + +if [ "$name" = "${STUB_DIE_AS:-}" ]; then + # Only die once every stub in $STUB_WAIT_FOR has its trap installed. Without + # this the entrypoint can reach its shutdown while a later stub is still + # starting, and that stub takes SIGTERM's default action instead of running + # its handler — a race in the test, not in the entrypoint. + for _ in $(seq 1 600); do + ready=1 + for other in ${STUB_WAIT_FOR:-}; do + if [ ! -f "/out/$other.ready" ]; then ready=0; fi + done + if [ "$ready" -eq 1 ]; then break; fi + sleep 0.1 + done + echo "stub[$name] exiting with ${STUB_DIE_CODE:-1}" + exit "${STUB_DIE_CODE:-1}" +fi +# `sleep & wait` rather than a bare `sleep`, so the trap runs as soon as the +# signal arrives instead of after the sleep returns. The loop matters: bash +# resumes after an interrupted `wait`, so without it even the trap that only +# logs would fall off the end of the script and exit. +while true; do + sleep 3000 & + wait $! +done +STUB + +cat > "$WORKDIR/harness.sh" <<'HARNESS' +#!/bin/bash +# Runs inside the container: installs the stubs, then drives the real +# /entrypoint.sh through each scenario. +set -uo pipefail + +mkdir -p /pyls /multiplayer /debugger /out +for tool in python3 node bun; do + cp /harness/stub "/usr/local/bin/$tool" + chmod +x "/usr/local/bin/$tool" +done + +export ENABLE_LSP=true ENABLE_MULTIPLAYER=true ENABLE_DEBUGGER=true ENABLE_GATEWAY=true +# Every stub the dying one must see started before it exits. +export STUB_WAIT_FOR="lsp multiplayer debugger gateway" + +failures=0 +check() { + local what="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + echo " ok: $what is $expected" + else + echo " FAIL: $what: expected $expected, got $actual" + failures=$((failures + 1)) + fi +} + +check_log() { + local what="$1" pattern="$2" + if grep -qE "$pattern" /out/log; then + echo " ok: $what" + else + echo " FAIL: $what (no line matching /$pattern/)" + failures=$((failures + 1)) + fi +} + +stopped() { + [ -f "/out/$1.stopped" ] && echo yes || echo no +} + +# `wait` on a backgrounded entrypoint, with a watchdog that SIGKILLs it rather +# than letting a wedged shutdown block the run. Sets $wait_status. +wait_bounded() { + local pid="$1" secs="${2:-30}" watchdog + (sleep "$secs"; kill -KILL "$pid" 2>/dev/null) & + watchdog=$! + wait "$pid" + wait_status=$? + kill "$watchdog" 2>/dev/null + wait "$watchdog" 2>/dev/null +} + +# Block until the entrypoint has written $1, so no scenario acts on a script that +# has not reached the state it is about to be tested in. +await_log() { + local pattern="$1" + for _ in $(seq 1 600); do + grep -qE "$pattern" /out/log && return 0 + sleep 0.1 + done + echo " FAIL: timed out waiting for /$pattern/ in the entrypoint log" + failures=$((failures + 1)) + return 1 +} + +echo "== a service that exits takes the container down with it" +rm -rf /out && mkdir -p /out +STUB_DIE_AS=multiplayer STUB_DIE_CODE=3 timeout -k 5 60 bash /entrypoint.sh > /out/log 2>&1 +check "exit status" 3 "$?" +cat /out/log +check_log "the dead service is named in the log" 'ERROR: Multiplayer \(PID: [0-9]+\) has exited' +# The survivors were signalled and ran their own shutdown, rather than being left +# running or killed outright. +for name in lsp debugger gateway; do + check "$name stopped cleanly" yes "$(stopped "$name")" +done + +echo +echo "== a service that exits 0 is still a failure for the container" +rm -rf /out && mkdir -p /out +STUB_DIE_AS=gateway STUB_DIE_CODE=0 timeout -k 5 60 bash /entrypoint.sh > /out/log 2>&1 +check "exit status" 1 "$?" +check_log "the dead service is named in the log" 'ERROR: Gateway \(PID: [0-9]+\) has exited' + +echo +echo "== a service that ignores SIGTERM cannot hold the container open" +rm -rf /out && mkdir -p /out +STUB_DIE_AS=multiplayer STUB_DIE_CODE=3 STUB_IGNORE_TERM=lsp SHUTDOWN_GRACE_SECS=2 \ + timeout -k 5 60 bash /entrypoint.sh > /out/log 2>&1 +check "exit status" 3 "$?" +check_log "the wedged service is killed after the grace period" \ + 'WARNING: PID [0-9]+ did not stop within 2s, killing it' +check_log "the shutdown still completes" 'All services stopped' +# The ones that do honour SIGTERM still stop the polite way. +for name in debugger gateway; do + check "$name stopped cleanly" yes "$(stopped "$name")" +done + +echo +echo "== SIGTERM is still a clean shutdown" +rm -rf /out && mkdir -p /out +bash /entrypoint.sh > /out/log 2>&1 & +entrypoint_pid=$! +if await_log 'All enabled services started'; then + kill -TERM "$entrypoint_pid" + wait_bounded "$entrypoint_pid" + check "exit status" 0 "$wait_status" + for name in lsp multiplayer debugger gateway; do + check "$name stopped cleanly" yes "$(stopped "$name")" + done +else + kill -KILL "$entrypoint_pid" 2>/dev/null +fi + +echo +echo "== with no services enabled the entrypoint sleeps instead of exiting" +rm -rf /out && mkdir -p /out +ENABLE_LSP=false ENABLE_MULTIPLAYER=false ENABLE_DEBUGGER=false ENABLE_GATEWAY=false \ + bash /entrypoint.sh > /out/log 2>&1 & +entrypoint_pid=$! +if await_log 'Sleeping indefinitely'; then + # Staying up is the correct behaviour here, so there is nothing to wait for + # but the passage of time. + sleep 2 + check "still running" yes "$(kill -0 "$entrypoint_pid" 2>/dev/null && echo yes || echo no)" +fi +# SIGKILL, not SIGTERM: this branch blocks in a foreground `sleep infinity`, so +# bash defers the trap until it returns. That is pre-existing behaviour and is +# not what this test is about. +kill -KILL "$entrypoint_pid" 2>/dev/null +wait_bounded "$entrypoint_pid" + +echo +if [ "$failures" -eq 0 ]; then + echo "all entrypoint supervision checks passed" +else + echo "$failures check(s) failed" +fi +exit "$failures" +HARNESS + +echo "Running entrypoint supervision tests in $IMAGE" + +# --entrypoint: the windmill-extra image's own ENTRYPOINT is ["/entrypoint.sh"] +# (docker/DockerfileExtra), which would swallow the command and start the real +# services instead of the harness. Harmless for a plain base image. +DOCKER_RUN=(docker run --rm --init + --entrypoint bash + -v "$ENTRYPOINT:/entrypoint.sh:ro" + -v "$WORKDIR:/harness:ro" + "$IMAGE" /harness/harness.sh) + +# Bound the whole run where the host has a `timeout` (GNU coreutils; stock macOS +# ships none), so a stalled image pull cannot hang a CI job. Every scenario +# inside the container is bounded on its own regardless. +if command -v timeout >/dev/null 2>&1; then + timeout "${ENTRYPOINT_TEST_TIMEOUT:-900}" "${DOCKER_RUN[@]}" +else + "${DOCKER_RUN[@]}" +fi diff --git a/multiplayer/server.mjs b/multiplayer/server.mjs index ee3ef60277..87b20bd66e 100644 --- a/multiplayer/server.mjs +++ b/multiplayer/server.mjs @@ -168,6 +168,13 @@ const getYDoc = (docname) => { return doc } +/** + * Every error message is peer-controlled: `applyAwarenessUpdate` runs + * `JSON.parse` on the peer's bytes and V8 quotes the offending input back. Only + * printable ASCII survives, so nothing in it can forge or flood a log line. + */ +const describeError = (error) => String(error?.message ?? error).replace(/[^\x20-\x7e]+/g, ' ').slice(0, 200) + const send = (conn, message) => { if (conn.readyState === 1) { // WebSocket.OPEN conn.send(message, err => { if (err) console.error(err) }) @@ -176,6 +183,7 @@ const send = (conn, message) => { const setupWSConnection = (conn, req, docName, bufferedMessages = []) => { const doc = getYDoc(docName) + const clientIp = req.socket.remoteAddress // Initialize awareness if (!doc.awareness) { @@ -189,22 +197,36 @@ const setupWSConnection = (conn, req, docName, bufferedMessages = []) => { doc.conns.add(conn) const messageHandler = (message) => { - const data = new Uint8Array(message) - const decoder = decoding.createDecoder(data) - const messageType = decoding.readVarUint(decoder) + // A frame that arrived before we decided to close this connection (or that a + // replay below already made moot) must not still be applied to the document. + if (conn.readyState !== 1) return // WebSocket.OPEN - switch (messageType) { - case messageSync: - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - syncProtocol.readSyncMessage(decoder, encoder, doc, null) - if (encoding.length(encoder) > 1) { - send(conn, encoding.toUint8Array(encoder)) + try { + const data = new Uint8Array(message) + const decoder = decoding.createDecoder(data) + const messageType = decoding.readVarUint(decoder) + + switch (messageType) { + case messageSync: { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.readSyncMessage(decoder, encoder, doc, null) + if (encoding.length(encoder) > 1) { + send(conn, encoding.toUint8Array(encoder)) + } + break } - break - case messageAwareness: - awarenessProtocol.applyAwarenessUpdate(awareness, decoding.readVarUint8Array(decoder), conn) - break + case messageAwareness: + awarenessProtocol.applyAwarenessUpdate(awareness, decoding.readVarUint8Array(decoder), conn) + break + } + } catch (error) { + // The decoders throw on unparseable input and `ws` re-emits a listener's + // exception on the process, so one bad frame from one peer would end the + // server for every document and client. Drop only the offender, with RFC + // 6455's 1007; the frame itself is untrusted and is never logged. + console.warn(`[${new Date().toISOString()}] MALFORMED MESSAGE: doc="${docName}" from=${clientIp} error="${describeError(error)}"`) + conn.close(1007, 'Invalid message') } } conn.on('message', messageHandler) @@ -259,7 +281,12 @@ const setupWSConnection = (conn, req, docName, bufferedMessages = []) => { }) // Replay, in order, the messages that arrived while the token was being - // verified, now that the handlers above are in place. + // verified, now that the handlers above are in place. The log line is the only + // outside evidence that a frame took this path rather than the live handler, + // and marks a client that beat the JWKS fetch on a slow-starting instance. + if (bufferedMessages.length > 0) { + console.log(`[${new Date().toISOString()}] REPLAY: doc="${docName}" from=${clientIp} messages=${bufferedMessages.length}`) + } bufferedMessages.forEach(messageHandler) } @@ -287,6 +314,14 @@ const server = http.createServer((req, res) => { const wss = new WebSocketServer({ server }) +// `ws` forwards the HTTP server's errors here, and those are fatal: a failed +// listen leaves nothing to serve, and exiting 0 would read as a clean shutdown. +// `process.exitCode` rather than `process.exit()`, which truncates this line. +wss.on('error', (error) => { + console.error(`[${new Date().toISOString()}] WEBSOCKET SERVER ERROR: ${describeError(error)}`) + process.exitCode = 1 +}) + wss.on('connection', async (ws, req) => { let docName = req.url?.slice(1).split('?')[0] || 'unknown' @@ -297,6 +332,14 @@ wss.on('connection', async (ws, req) => { const clientIp = req.socket.remoteAddress + // A frame `ws` cannot parse at the protocol level fails in its Receiver, never + // reaching the handler below, and an unhandled 'error' on an EventEmitter ends + // the process. Attached before authentication, since the pre-auth window is + // exposed too; `ws` has already closed the connection by the time this runs. + ws.on('error', (error) => { + console.warn(`[${new Date().toISOString()}] SOCKET ERROR: doc="${docName}" from=${clientIp} error="${describeError(error)}"`) + }) + // Handle ping test — respond and close immediately if (docName === '__ping__') { console.log(`[${new Date().toISOString()}] WS ping from=${clientIp}`) diff --git a/multiplayer/test/cold_start.test.mjs b/multiplayer/test/cold_start.test.mjs index 4c1d37f92d..e2ac68efcb 100644 --- a/multiplayer/test/cold_start.test.mjs +++ b/multiplayer/test/cold_start.test.mjs @@ -15,13 +15,21 @@ import assert from 'node:assert/strict' import test from 'node:test' import crypto from 'node:crypto' import { setTimeout as delay } from 'node:timers/promises' -import { WebSocket } from 'ws' import * as Y from 'yjs' import * as syncProtocol from 'y-protocols/sync' import * as encoding from 'lib0/encoding' import * as decoding from 'lib0/decoding' import { mintToken, startJwksServer, startMultiplayerServer, waitFor } from './helpers.mjs' +import { + hasSyncType, + messageKind, + openClient, + syncStep1Message, + syncStep2, + syncUpdate, + updateMessage +} from './protocol.mjs' const WORKSPACE = 'test_workspace' const DOC_PATH = `${WORKSPACE}/f/foo/bar` @@ -33,53 +41,6 @@ const FLIGHT_MARGIN_MS = 250 // server demonstrably has no key, so anything it receives is in the pre-auth window. const KEY_LOADED = 'Successfully loaded Ed25519 public key' -const messageSync = 0 -const syncStep2 = 1 - -/** `messageSync` + sync step 1 for an empty document, as y-websocket sends on open. */ -function syncStep1Message() { - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - syncProtocol.writeSyncStep1(encoder, new Y.Doc()) - return encoding.toUint8Array(encoder) -} - -/** `messageSync` + a Yjs update inserting `text` into the 'content' text type. */ -function updateMessage(text) { - const doc = new Y.Doc() - doc.getText('content').insert(0, text) - const encoder = encoding.createEncoder() - encoding.writeVarUint(encoder, messageSync) - syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(doc)) - return encoding.toUint8Array(encoder) -} - -/** Read the (messageType, syncType) pair that prefixes a sync message. */ -function messageKind(data) { - const decoder = decoding.createDecoder(data) - const messageType = decoding.readVarUint(decoder) - if (messageType !== messageSync) return { messageType } - return { messageType, syncType: decoding.readVarUint(decoder) } -} - -/** Open a socket and collect every frame it receives, plus its close code. */ -function openClient(url, { onOpen } = {}) { - const ws = new WebSocket(url) - const received = [] - const client = { ws, received, closeCode: undefined } - ws.on('message', (data) => received.push(new Uint8Array(data))) - ws.on('close', (code) => { - client.closeCode = code - }) - ws.on('error', () => {}) - if (onOpen) ws.on('open', () => onOpen(ws)) - return client -} - -function hasKind(client, syncType) { - return client.received.some((data) => messageKind(data).syncType === syncType) -} - test('cold start: a sync step 1 sent while the key is still being fetched is answered', { timeout: 60000 }, async (t) => { const jwks = await startJwksServer({ hold: true }) const server = await startMultiplayerServer({ WINDMILL_BASE_URL: jwks.baseUrl }) @@ -112,11 +73,11 @@ test('cold start: a sync step 1 sent while the key is still being fetched is ans await delay(FLIGHT_MARGIN_MS) jwks.release() - await waitFor(() => hasKind(cold, syncStep2), { + await waitFor(() => hasSyncType(cold, syncStep2), { message: 'the server to answer the cold client with sync step 2' }) // The echo of our own update proves it was applied to the server-side doc. - await waitFor(() => hasKind(cold, 2), { + await waitFor(() => hasSyncType(cold, syncUpdate), { message: 'the server to broadcast back the update sent during the cold window' }) cold.ws.close() @@ -127,7 +88,7 @@ test('cold start: a sync step 1 sent while the key is still being fetched is ans }) t.after(() => warm.ws.close()) - await waitFor(() => hasKind(warm, syncStep2), { + await waitFor(() => hasSyncType(warm, syncStep2), { message: 'the server to answer the second client with sync step 2' }) @@ -196,7 +157,7 @@ test('cold start: a peer that floods before authenticating is closed and the ser }) t.after(() => healthy.ws.close()) - await waitFor(() => hasKind(healthy, syncStep2), { + await waitFor(() => hasSyncType(healthy, syncStep2), { message: 'the server to still answer a well-behaved client with sync step 2' }) }) diff --git a/multiplayer/test/helpers.mjs b/multiplayer/test/helpers.mjs index e998cd4961..371123d864 100644 --- a/multiplayer/test/helpers.mjs +++ b/multiplayer/test/helpers.mjs @@ -172,6 +172,12 @@ export async function startMultiplayerServer(env = {}, attemptsLeft = 5) { get output() { return output }, + /** How the child died, or null while it is still running. */ + get exitStatus() { + if (child.exitCode !== null) return `exited with code ${child.exitCode}` + if (child.signalCode !== null) return `killed by ${child.signalCode}` + return null + }, async close() { if (child.exitCode !== null) return const exited = new Promise((resolve) => child.once('exit', resolve)) @@ -181,6 +187,52 @@ export async function startMultiplayerServer(env = {}, attemptsLeft = 5) { } } +/** + * Start server.mjs and wait for it to exit, for the cases where it is expected + * to fail rather than come up. Returns how it died and everything it printed. + */ +export async function runMultiplayerServerUntilExit(env = {}, { timeoutMs = 15000 } = {}) { + const child = spawn(process.execPath, [SERVER_PATH], { + env: { + ...process.env, + REQUIRE_SIGNED_MULTIPLAYER_REQUESTS: 'true', + BASE_INTERNAL_URL: '', + HOST: '127.0.0.1', + ...env + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + + let output = '' + child.stdout.on('data', (chunk) => { output += chunk.toString() }) + child.stderr.on('data', (chunk) => { output += chunk.toString() }) + + // Record the rescue rather than inferring it from the exit signal: `signal` is + // null for every child exit on Windows, so a caller checking it could not tell + // a server that exited on its own from one this had to kill. + let killedByTimeout = false + const killer = setTimeout(() => { + // Claim the rescue only if there was something to rescue. The child may have + // exited moments ago with 'close' still pending on the stdio drain — the very + // window this helper waits for — and killing a corpse is not an intervention. + if (child.exitCode !== null || child.signalCode !== null) return + killedByTimeout = child.kill('SIGKILL') + }, timeoutMs) + + try { + // 'close' rather than 'exit': 'exit' fires when the child terminates, which + // can be before its stdio pipes have been drained, and the caller reads + // `output`. A child that never starts emits neither, only 'error'. + const [code, signal] = await new Promise((resolve, reject) => { + child.once('close', (exitCode, exitSignal) => resolve([exitCode, exitSignal])) + child.once('error', reject) + }) + return { code, signal, output, killedByTimeout } + } finally { + clearTimeout(killer) + } +} + /** Poll `predicate` until it is true, or throw after `timeoutMs`. */ export async function waitFor(predicate, { timeoutMs = 15000, intervalMs = 10, message = 'condition' } = {}) { const deadline = Date.now() + timeoutMs diff --git a/multiplayer/test/malformed_frame.test.mjs b/multiplayer/test/malformed_frame.test.mjs new file mode 100644 index 0000000000..8ba340de37 --- /dev/null +++ b/multiplayer/test/malformed_frame.test.mjs @@ -0,0 +1,231 @@ +/** + * Regression tests for malformed frames sent by an authenticated peer. + * + * The 'message' handler in `setupWSConnection` decodes whatever arrives with + * `decoding.readVarUint`, `syncProtocol.readSyncMessage` and + * `awarenessProtocol.applyAwarenessUpdate`. All three throw on input they cannot + * parse, and `ws` emits the listener's exception on the process. With no + * `uncaughtException` handler installed, one such frame used to take the whole + * multiplayer server down — every other document and every other client with it. + * + * The payloads below are the ones that reproduce it: 600 KiB of zeros decodes as + * sync step 1 with an empty state vector, which makes `Y.encodeStateAsUpdate` + * throw "Unexpected end of array"; the truncated frames run off the end of the + * buffer inside the decoder. + */ + +import assert from 'node:assert/strict' +import test from 'node:test' +import { setTimeout as delay } from 'node:timers/promises' +import * as encoding from 'lib0/encoding' + +import { mintToken, startJwksServer, startMultiplayerServer, waitFor } from './helpers.mjs' +import { + hasSyncType, + messageAwareness, + messageSync, + openClient, + syncedDoc, + syncStep1, + syncStep1Message, + syncStep2, + updateMessage +} from './protocol.mjs' + +const WORKSPACE = 'test_workspace' +const DOC_PATH = `${WORKSPACE}/f/foo/bar` +// server.mjs closes a connection whose frame it could not decode with 1007 +// ("invalid frame payload data"). +const INVALID_PAYLOAD = 1007 +// Logged by server.mjs for every frame it refused; asserted on so the tests +// cannot pass on a connection that was closed for some unrelated reason. +const REFUSED = 'MALFORMED MESSAGE' +// Logged by server.mjs once the JWKS fetch resolves. While it is absent the +// server demonstrably has no key, so anything it received is in the pre-auth window. +const KEY_LOADED = 'Successfully loaded Ed25519 public key' +// Grace for a frame the client has already written to be delivered over loopback +// and read by the (otherwise idle) server, before the key is released. Only the +// already-written bytes have to land in this time, and the REPLAY assertion +// below fails loudly if they did not, rather than quietly testing the wrong path. +const FLIGHT_MARGIN_MS = 500 +// Logged by server.mjs when, and only when, it replays messages that were +// buffered before authentication. +const REPLAYED = 'REPLAY:' +// A control character a peer would use to forge log lines or drive a terminal. +const ESC = '\x1b' + +/** + * 600 KiB of zeros: `messageSync`, then sync step 1 with a zero-length state + * vector, which `Y.encodeStateAsUpdate` cannot decode. + */ +function zeroFlood() { + return Buffer.alloc(600 * 1024) +} + +/** `messageSync` with nothing after it: `readSyncMessage` reads past the end. */ +function truncatedSyncMessage() { + return Uint8Array.from([messageSync]) +} + +/** `messageSync` + sync step 1 whose state vector is shorter than it claims. */ +function truncatedStateVector() { + return Uint8Array.from([messageSync, syncStep1, 8, 1, 2, 3]) +} + +/** `messageAwareness` + an awareness payload shorter than it claims. */ +function truncatedAwarenessMessage() { + return Uint8Array.from([messageAwareness, 8, 1, 2, 3]) +} + +/** + * `messageAwareness` + one well-formed entry whose state is not JSON, so + * `applyAwarenessUpdate` fails inside `JSON.parse`. V8 quotes the offending + * input back in its message, which is how peer bytes — ESC included — can reach + * a log line that interpolates `error.message`. + */ +function awarenessWithUnparseableState() { + const update = encoding.createEncoder() + encoding.writeVarUint(update, 1) // one client + encoding.writeVarUint(update, 42) // client id + encoding.writeVarUint(update, 1) // clock + // Not starting with '{': V8 only quotes the input back for a value that is + // invalid from position 0, which is the case this needs to reproduce. + encoding.writeVarString(update, `x${ESC}[2J OWNED THE LOG`) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageAwareness) + encoding.writeVarUint8Array(encoder, encoding.toUint8Array(update)) + return encoding.toUint8Array(encoder) +} + +const MALFORMED = [ + ['600 KiB of zeros', zeroFlood], + ['a truncated sync message', truncatedSyncMessage], + ['a truncated sync step 1 state vector', truncatedStateVector], + ['a truncated awareness update', truncatedAwarenessMessage], + ['an awareness state that is not JSON', awarenessWithUnparseableState] +] + +/** Connect, wait for the server's sync step 1, i.e. for the peer to be authenticated. */ +async function connectAuthenticated(server, token, { onOpen } = {}) { + const client = openClient(`${server.url}/${DOC_PATH}?token=${token}`, { onOpen }) + await waitFor(() => hasSyncType(client, syncStep1) || client.closeCode !== undefined, { + message: 'the server to send sync step 1' + }) + assert.equal(client.closeCode, undefined, 'connection was closed before it was set up') + return client +} + +for (const [label, payload] of MALFORMED) { + test(`an authenticated client sending ${label} is closed, and the server survives`, { timeout: 60000 }, async (t) => { + const jwks = await startJwksServer() + const server = await startMultiplayerServer({ WINDMILL_BASE_URL: jwks.baseUrl }) + t.after(async () => { + await server.close() + await jwks.close() + }) + + const token = mintToken(jwks.privateKey, { workspaceId: WORKSPACE }) + + // A bystander on the same document, synced before anything goes wrong. + const bystander = await connectAuthenticated(server, token, { + onOpen: (ws) => ws.send(syncStep1Message()) + }) + await waitFor(() => hasSyncType(bystander, syncStep2), { + message: 'the server to answer the bystander with sync step 2' + }) + t.after(() => bystander.ws.close()) + + // The offender is fully authenticated — the server has already answered it + // with sync step 1 — so this frame goes through the live message handler, + // not the pre-auth buffer. + const offender = await connectAuthenticated(server, token) + offender.ws.send(payload()) + + await waitFor(() => offender.closeCode !== undefined, { + message: 'the offending connection to be closed' + }) + assert.equal(offender.closeCode, INVALID_PAYLOAD) + assert.equal(server.exitStatus, null, `server died: ${server.output}`) + // The frame itself is never logged: 600 KiB of zeros must not reach the log. + assert.ok(server.output.length < 8192, `server logged ${server.output.length} bytes, payload leaked?`) + + // Exactly one refusal line, and every assertion about it is made against the + // line itself: CONNECT and DISCONNECT also name the document, so checking the + // whole log would pass even if the refusal stopped naming anything. + const refusals = server.output.split('\n').filter((line) => line.includes(REFUSED)) + assert.equal(refusals.length, 1, `expected one refusal line, got:\n${server.output}`) + assert.ok(refusals[0].includes(`doc="${DOC_PATH}"`), `the refusal must name the document: ${refusals[0]}`) + assert.ok(refusals[0].includes('from='), `the refusal must name the peer: ${refusals[0]}`) + // Nothing a peer put in the frame can escape that line: an error message can + // quote the payload (V8 does so for JSON.parse), so it must carry no control + // characters at all. + assert.doesNotMatch(refusals[0], /[\x00-\x1f\x7f]/, 'the refusal line must have no control characters') + + // The bystander was not disturbed, and a new client still syncs. + assert.equal(bystander.closeCode, undefined) + const latecomer = await connectAuthenticated(server, token, { + onOpen: (ws) => ws.send(syncStep1Message()) + }) + t.after(() => latecomer.ws.close()) + await waitFor(() => hasSyncType(latecomer, syncStep2), { + message: 'the server to answer a new client with sync step 2' + }) + + // And a real edit still propagates from one client to the other. + const bystanderDoc = syncedDoc(bystander) + latecomer.ws.send(updateMessage('hello')) + await waitFor(() => bystanderDoc().getText('content').toString() === 'hello', { + message: 'the edit to propagate to the bystander' + }) + }) +} + +test('a malformed frame replayed from the pre-auth buffer is refused, not fatal', { timeout: 60000 }, async (t) => { + const jwks = await startJwksServer({ hold: true }) + const server = await startMultiplayerServer({ WINDMILL_BASE_URL: jwks.baseUrl }) + t.after(async () => { + await server.close() + await jwks.close() + }) + + const token = mintToken(jwks.privateKey, { workspaceId: WORKSPACE }) + + // The key is parked, so this frame is buffered by the pre-auth path and only + // decoded later, when `setupWSConnection` replays it. One 600 KiB frame stays + // under MAX_PREAUTH_BYTES (1 MiB), so the flood cap cannot be what closes it. + await waitFor(() => jwks.requests >= 1, { message: 'the server to request the JWKS' }) + let framesWritten = 0 + const offender = openClient(`${server.url}/${DOC_PATH}?token=${token}`, { + onOpen: (ws) => ws.send(zeroFlood(), () => framesWritten++) + }) + await waitFor(() => framesWritten === 1, { message: 'the offending client to write its frame' }) + // The frame is on the wire while the server demonstrably has no key, so it + // cannot have been authenticated yet. + assert.ok(!server.output.includes(KEY_LOADED)) + await delay(FLIGHT_MARGIN_MS) + jwks.release() + + await waitFor(() => offender.closeCode !== undefined, { + message: 'the offending connection to be closed' + }) + assert.equal(offender.closeCode, INVALID_PAYLOAD) + // Without this the test would pass just as well if the frame had arrived after + // setup and gone through the live handler — the same close code, the same log + // line — so it would silently stop covering the replay path it is named for. + // server.mjs logs REPLAY only when it replays a pre-auth buffer. + assert.ok( + server.output.includes(`${REPLAYED} doc="${DOC_PATH}" from=127.0.0.1 messages=1`), + `the frame did not go through the pre-auth buffer:\n${server.output}` + ) + assert.equal(server.exitStatus, null, `server died: ${server.output}`) + assert.ok(server.output.includes(REFUSED), `server did not log the refused frame:\n${server.output}`) + + // The server is unharmed and still syncs a well-behaved client. + const healthy = await connectAuthenticated(server, token, { + onOpen: (ws) => ws.send(syncStep1Message()) + }) + t.after(() => healthy.ws.close()) + await waitFor(() => hasSyncType(healthy, syncStep2), { + message: 'the server to still answer a well-behaved client with sync step 2' + }) +}) diff --git a/multiplayer/test/protocol.mjs b/multiplayer/test/protocol.mjs new file mode 100644 index 0000000000..77aeea3db9 --- /dev/null +++ b/multiplayer/test/protocol.mjs @@ -0,0 +1,94 @@ +/** + * y-websocket protocol helpers shared by the multiplayer server tests: + * building the frames a real client sends, reading back what the server + * answers, and a socket wrapper that records both. + */ + +import { WebSocket } from 'ws' +import * as Y from 'yjs' +import * as syncProtocol from 'y-protocols/sync' +import * as encoding from 'lib0/encoding' +import * as decoding from 'lib0/decoding' + +// Message types, as server.mjs numbers them. +export const messageSync = 0 +export const messageAwareness = 1 + +// y-protocols/sync sub-types. These overlap numerically with the message types +// above (`syncStep1 === messageSync === 0`), so no runtime check can catch a +// caller that passes the wrong family: `hasSyncType` takes only these three. +export const syncStep1 = 0 +export const syncStep2 = 1 +export const syncUpdate = 2 + +/** `messageSync` + sync step 1 for an empty document, as y-websocket sends on open. */ +export function syncStep1Message() { + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeSyncStep1(encoder, new Y.Doc()) + return encoding.toUint8Array(encoder) +} + +/** `messageSync` + a Yjs update inserting `text` into the 'content' text type. */ +export function updateMessage(text) { + const doc = new Y.Doc() + doc.getText('content').insert(0, text) + const encoder = encoding.createEncoder() + encoding.writeVarUint(encoder, messageSync) + syncProtocol.writeUpdate(encoder, Y.encodeStateAsUpdate(doc)) + return encoding.toUint8Array(encoder) +} + +/** Read the (messageType, syncType) pair that prefixes a sync message. */ +export function messageKind(data) { + const decoder = decoding.createDecoder(data) + const messageType = decoding.readVarUint(decoder) + if (messageType !== messageSync) return { messageType } + return { messageType, syncType: decoding.readVarUint(decoder) } +} + +/** + * A live mirror of what `client` has been sent: one document that each call + * brings up to date with the frames received since the last one. Cheap enough + * to call from a polling predicate, and never re-applies a frame. + */ +export function syncedDoc(client) { + const doc = new Y.Doc() + let applied = 0 + return () => { + while (applied < client.received.length) { + const data = client.received[applied++] + if (messageKind(data).messageType !== messageSync) continue + const decoder = decoding.createDecoder(data) + decoding.readVarUint(decoder) // messageSync + syncProtocol.readSyncMessage(decoder, encoding.createEncoder(), doc, null) + } + return doc + } +} + +/** Open a socket and collect every frame it receives, plus its close code. */ +export function openClient(url, { onOpen } = {}) { + const ws = new WebSocket(url) + const received = [] + const client = { ws, received, closeCode: undefined, lastError: undefined } + // Under the default binaryType 'nodebuffer' `ws` hands every frame over as a + // Buffer, text frames included, so this is lossless for both. + ws.on('message', (data) => received.push(new Uint8Array(data))) + ws.on('close', (code) => { + client.closeCode = code + }) + // Keep the error rather than only silencing it: without a listener `ws` would + // throw, and without the record a failed connection shows up as a `waitFor` + // timeout with no cause attached. + ws.on('error', (error) => { + client.lastError = error + }) + if (onOpen) ws.on('open', () => onOpen(ws)) + return client +} + +/** Has `client` received a sync message of this sub-type (never a message type)? */ +export function hasSyncType(client, syncType) { + return client.received.some((data) => messageKind(data).syncType === syncType) +} diff --git a/multiplayer/test/socket_error.test.mjs b/multiplayer/test/socket_error.test.mjs new file mode 100644 index 0000000000..4978906a50 --- /dev/null +++ b/multiplayer/test/socket_error.test.mjs @@ -0,0 +1,128 @@ +/** + * Regression tests for WebSocket-level protocol errors. + * + * A frame the `ws` library itself cannot parse — an unmasked frame from a + * client, a reserved opcode, a bad RSV bit — never reaches the application's + * 'message' handler. `ws` fails it in its Receiver and emits an 'error' on the + * WebSocket (`receiverOnError` in ws/lib/websocket.js), and an unhandled 'error' + * on an EventEmitter throws, so the process used to exit here too — the same + * blast radius as a malformed application payload, reached one layer lower. + * + * Raw bytes are written straight to the TCP socket, since the `ws` client would + * never produce an illegal frame on its own. + */ + +import assert from 'node:assert/strict' +import net from 'node:net' +import test from 'node:test' + +import { + mintToken, + runMultiplayerServerUntilExit, + startJwksServer, + startMultiplayerServer, + waitFor +} from './helpers.mjs' +import { hasSyncType, openClient, syncStep1, syncStep1Message, syncStep2 } from './protocol.mjs' + +const WORKSPACE = 'test_workspace' +const DOC_PATH = `${WORKSPACE}/f/foo/bar` +// Logged by server.mjs for every connection `ws` failed at the protocol level. +const SOCKET_ERROR = 'SOCKET ERROR' +// RFC 6455 "protocol error": what `ws` closes with when its Receiver rejects a +// frame, carried on the Receiver's error as `Symbol(status-code)`. +const PROTOCOL_ERROR = 1002 + +/** + * A FIN + text frame of 3 bytes with the MASK bit clear. RFC 6455 requires every + * client-to-server frame to be masked, so `ws` rejects it with + * WS_ERR_EXPECTED_MASK (close status 1002). + */ +const UNMASKED_FRAME = Buffer.from([0x81, 0x03, 0x61, 0x62, 0x63]) + +/** Assert the server is alive by making it serve a fresh client. */ +async function assertStillServing(t, server, token) { + const healthy = openClient(`${server.url}/${DOC_PATH}?token=${token}`, { + onOpen: (ws) => ws.send(syncStep1Message()) + }) + t.after(() => healthy.ws.close()) + // Resolve as soon as either outcome is settled, so a dead server fails fast + // and with its own output rather than by timing out. + await waitFor(() => hasSyncType(healthy, syncStep2) || server.exitStatus !== null, { + message: 'the server to answer a new client with sync step 2' + }) + assert.equal(server.exitStatus, null, `server died: ${server.output}`) + assert.ok(hasSyncType(healthy, syncStep2)) +} + +test('an illegal WebSocket frame from an authenticated client does not exit the server', { timeout: 60000 }, async (t) => { + const jwks = await startJwksServer() + const server = await startMultiplayerServer({ WINDMILL_BASE_URL: jwks.baseUrl }) + t.after(async () => { + await server.close() + await jwks.close() + }) + + const token = mintToken(jwks.privateKey, { workspaceId: WORKSPACE }) + + const offender = openClient(`${server.url}/${DOC_PATH}?token=${token}`) + // The server's sync step 1 means this peer is past authentication. + await waitFor(() => hasSyncType(offender, syncStep1), { message: 'the server to send sync step 1' }) + offender.ws._socket.write(UNMASKED_FRAME) + + await waitFor(() => offender.closeCode !== undefined, { + message: 'the offending connection to be closed' + }) + assert.equal(offender.closeCode, PROTOCOL_ERROR) + assert.equal(server.exitStatus, null, `server died: ${server.output}`) + assert.ok(server.output.includes(SOCKET_ERROR), `server did not log the socket error:\n${server.output}`) + + await assertStillServing(t, server, token) +}) + +test('an illegal WebSocket frame before authentication does not exit the server', { timeout: 60000 }, async (t) => { + const jwks = await startJwksServer({ hold: true }) + const server = await startMultiplayerServer({ WINDMILL_BASE_URL: jwks.baseUrl }) + t.after(async () => { + await server.close() + await jwks.close() + }) + + const token = mintToken(jwks.privateKey, { workspaceId: WORKSPACE }) + + // The JWKS response is parked and never released before the assertions below, + // so the server cannot authenticate anyone for the whole of this test. + const offender = openClient(`${server.url}/${DOC_PATH}?token=${token}`, { + onOpen: (ws) => ws._socket.write(UNMASKED_FRAME) + }) + + await waitFor(() => offender.closeCode !== undefined, { + message: 'the offending connection to be closed' + }) + assert.equal(offender.closeCode, PROTOCOL_ERROR) + // server.mjs logs CONNECT only once a peer is past verification, so its absence + // is what makes this the pre-auth case rather than a repeat of the test above. + assert.ok(!server.output.includes('CONNECT:'), `a connection was accepted:\n${server.output}`) + assert.equal(server.exitStatus, null, `server died: ${server.output}`) + assert.ok(server.output.includes(SOCKET_ERROR), `server did not log the socket error:\n${server.output}`) + + jwks.release() + await assertStillServing(t, server, token) +}) + +test('a server-level error is fatal, not swallowed', { timeout: 60000 }, async (t) => { + // Hold the port so server.mjs's listen fails with EADDRINUSE. `ws` forwards + // the HTTP server's errors to the WebSocketServer, so this is what reaches the + // `wss.on('error')` handler — and a process with no listening socket must not + // report a clean exit, or nothing upstream knows to replace it. + const blocker = net.createServer() + await new Promise((resolve) => blocker.listen(0, '127.0.0.1', resolve)) + const { port } = blocker.address() + t.after(() => new Promise((resolve) => blocker.close(resolve))) + + const { code, output, killedByTimeout } = await runMultiplayerServerUntilExit({ PORT: String(port) }) + + assert.equal(killedByTimeout, false, `the server had to be killed rather than exiting:\n${output}`) + assert.ok(output.includes('EADDRINUSE'), `expected a listen failure, got:\n${output}`) + assert.notEqual(code, 0, `expected a non-zero exit, got ${code}:\n${output}`) +}) From 3bd89e92d8f563b564a3d2e619749f377a7d6982 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 25 Sep 2026 17:12:51 +0200 Subject: [PATCH 29/91] feat: collapse the fork members setting and show its state in a badge (#11220) * feat: collapse the fork members setting and show its state in a badge Co-Authored-By: Claude Opus 5 * fix: show the fork members badge next to the title, only when on, like other section badges Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../lib/components/settings/WorkspaceUserSettings.svelte | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte index 449e9c7d31..8f5db5f2d9 100644 --- a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte +++ b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte @@ -1091,7 +1091,13 @@ label="Fork members" description="Choose who a new fork of this workspace starts with, besides the person who creates it." wrapperClass="pb-12" + collapsable > + {#snippet badge()} + {#if addAdminsAndDevelopersToForks} + Admins and developers + {/if} + {/snippet} updateAddAdminsAndDevelopersToForks(e.detail)} From f183bd43fbfcc288a8bd8ce097296777b96ca2fc Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 25 Sep 2026 11:13:06 -0400 Subject: [PATCH 30/91] chore: retire the standalone lsp and multiplayer images from examples, drop lsp/Dockerfile (#11341) * chore: retire the standalone lsp and multiplayer images from examples, drop lsp/Dockerfile Co-Authored-By: Claude Opus 5.5 (1M context) * chore: drop the unbuilt DockerfileMultiplayer, document running the LSP from windmill-extra Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): ecs terraform destroys cleanly and gives private instances no public ip Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): give windmill-extra on ecs a WINDMILL_BASE_URL for multiplayer auth, address review Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): make the ecs example upgrade cleanly from the standalone lsp/multiplayer stack Co-Authored-By: Claude Opus 5.5 (1M context) * fix(examples): name the extra target group by prefix so create_before_destroy can replace it Co-Authored-By: Claude Opus 5.5 (1M context) * docs(examples): note the brief editor-socket gap when upgrading the ecs example Co-Authored-By: Claude Opus 5.5 (1M context) * docs(examples): the debugger stays off after the ecs upgrade unless enabled Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- docker/DockerfileMultiplayer | 9 -- docs/docker-security.md | 5 +- examples/deploy/aws-ecs-terraform/README.md | 18 ++- .../deploy/aws-ecs-terraform/ecs_cluster.tf | 17 ++- .../deploy/aws-ecs-terraform/load_balancer.tf | 50 ++++--- examples/deploy/aws-ecs-terraform/rds.tf | 9 +- examples/deploy/aws-ecs-terraform/vpc.tf | 6 + .../aws-ecs-terraform/windmill_extra.tf | 126 ++++++++++++++++++ .../deploy/aws-ecs-terraform/windmill_lsp.tf | 85 ------------ .../aws-ecs-terraform/windmill_multiplayer.tf | 80 ----------- .../windmill_worker_high_performance.tf | 17 ++- .../aws-eks-cloudformation/quicklaunch.yaml | 2 +- .../otel-tracing-grafana/docker-compose.yml | 33 +++-- .../otel-tracing-jaeger/docker-compose.yml | 33 +++-- frontend/README_DEV.md | 18 +++ lsp/Dockerfile | 62 --------- lsp/dev.nu | 21 --- 17 files changed, 262 insertions(+), 329 deletions(-) delete mode 100644 docker/DockerfileMultiplayer create mode 100644 examples/deploy/aws-ecs-terraform/windmill_extra.tf delete mode 100644 examples/deploy/aws-ecs-terraform/windmill_lsp.tf delete mode 100644 examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf delete mode 100644 lsp/Dockerfile delete mode 100755 lsp/dev.nu diff --git a/docker/DockerfileMultiplayer b/docker/DockerfileMultiplayer deleted file mode 100644 index 192ac06ff5..0000000000 --- a/docker/DockerfileMultiplayer +++ /dev/null @@ -1,9 +0,0 @@ -FROM node:slim - -EXPOSE 3002 -ENV PORT 3002 -ENV HOST 0.0.0.0 - -RUN npm install -g y-websocket - -CMD ["npx", "y-websocket"] \ No newline at end of file diff --git a/docs/docker-security.md b/docs/docker-security.md index 5b5196f8b4..44ae6cbeff 100644 --- a/docs/docker-security.md +++ b/docs/docker-security.md @@ -22,9 +22,8 @@ so patching here is sufficient: - `DockerfileExtra` builds `FROM windmill-ee-slim`. The nsjail *builder* stages are throwaway (only the compiled `nsjail` binary is -copied out), so they are intentionally not upgraded. `DockerfileMultiplayer` -(`node:slim`), the CLI, Caddy-L4, CUDA-only, and RHEL/dnf images are out of scope -for this apt-based patching. +copied out), so they are intentionally not upgraded. The CLI, Caddy-L4, +CUDA-only, and RHEL/dnf images are out of scope for this apt-based patching. ## Why `apt-get upgrade` and not `unattended-upgrades` / pinning diff --git a/examples/deploy/aws-ecs-terraform/README.md b/examples/deploy/aws-ecs-terraform/README.md index 098943e353..f9a0318d7a 100644 --- a/examples/deploy/aws-ecs-terraform/README.md +++ b/examples/deploy/aws-ecs-terraform/README.md @@ -32,14 +32,13 @@ Windmill relies on a PostgreSQL database which will be created by Terraform. The The following stack will be deployed by this terraform as it is provided: - 2 Windmill servers (each requiring 1 CPU and 1.5GiB of Memory) - 2 multi-purpose Windmill workers (each requiring 2 CPU and 3GiB of Memory) -- Windmill LSP (requiring 1 CPU and 1.5GiB of Memory) -- Windmill Multiplayer (requiring 1 CPU and 1.5GiB of Memory) +- Windmill Extra: LSP and Multiplayer, plus the optional Debugger (requiring 1 CPU and 1.5GiB of Memory) - 1 native Windmill worker (requiring 2 CPU and 3GiB of Memory) - 1 high performance Windmill worker (requiring 4 CPU and 15GiB of Memory) -The ECS cluster will be composed of 1 autoscaling group composed of up-to 6 `t3.medium` instances (5 necessary for the entire load, and 1 more for rolling upgrades). This will host all the services except the high performance workers. Those will be deployed on a separate auto-scaling group composed of up-to 2 `t3.xlarge` instances. +The ECS cluster will be composed of 1 autoscaling group of `t3.medium` instances, sized by the ECS capacity provider up to `max_size` (10) in [ecs_cluster.tf](./ecs_cluster.tf). In our test deployment it settled on 5 instances for this load; the headroom covers rolling upgrades. This will host all the services except the high performance workers. Those will be deployed on a separate auto-scaling group composed of up-to 2 `t3.xlarge` instances. -Of course the above is provided as an example, it should be tuned for you own needs, both in terms of instance specs, but also in terms of service architecture. For example, you might not need high performance workers, in which case you won't need the second autoscaling group at all. Same for the native worker, multiplayer, or even LSP (though LSP is highly recommended for a better coding experience). +Of course the above is provided as an example, it should be tuned for your own needs, both in terms of instance specs, but also in terms of service architecture. For example, you might not need high performance workers, in which case you won't need the second autoscaling group at all. Same for the native worker, or even Windmill Extra (though its LSP is highly recommended for a better coding experience). Its services are switched on and off with the `ENABLE_LSP`, `ENABLE_MULTIPLAYER` (Enterprise Edition) and `ENABLE_DEBUGGER` variables in [windmill_extra.tf](./windmill_extra.tf). The only strictly required components are: - At least 1 Windmill server @@ -49,14 +48,13 @@ This terraform has been assembled to easily remove components. Each component me - _REQUIRED_ Windmill server -> [windmill_server.tf](./windmill_server.tf) - _REQUIRED_ Multi-purpose windmill worker -> [windmill_worker_basic.tf](./windmill_worker_basic.tf) -- _OPTIONAL_ Windmill LSP -> [windmill_lsp.tf](./windmill_lsp.tf) -- _OPTIONAL_ Windmill Multiplayer -> [windmill_multiplayer.tf](./windmill_multiplayer.tf) +- _OPTIONAL_ Windmill Extra (LSP, Multiplayer, Debugger) -> [windmill_extra.tf](./windmill_extra.tf) - _OPTIONAL_ Windmill Native worker -> [windmill_worker_native.tf](./windmill_worker_native.tf) - _OPTIONAL_ Windmill High Performace worker -> [windmill_worker_high_performance.tf](./windmill_worker_high_performance.tf) - this one is a lot longer because it deploys a new auto scaling group ### Network -The network deployed here is a single VPC, composed on 4 subnets spread across 2 availability zones (1 public and 1 private subnet per zone). All the services are behind a load balancer routing the request to Windmill server, LSP or multiplayer. +The network deployed here is a single VPC, composed on 4 subnets spread across 2 availability zones (1 public and 1 private subnet per zone). All the services are behind a load balancer routing the request to Windmill server or Windmill Extra (`/ws/*`, `/ws_mp/*`, `/ws_debug/*`). A single security group is used, allowing HTTP traffic on port 80 (it is not deploying custom certificates and therefore does not use HTTPS). @@ -66,6 +64,12 @@ All this is provided as an example. It can be refined depending on your needs. A Windmill heavily relies on PostgreSQL database. This terraform deploys a standalone RDS instance having 100GiB of storage, which can be scaled up to 1000GiB. For production use cases, we recommend deploying a Multi-AZ instance, or even a Multi-AZ DB cluster. The RDS definition is in [rds.tf](./rds.tf) +### Upgrading a stack deployed before Windmill Extra + +Earlier versions of this example ran the LSP and Multiplayer as two separate services (`windmill_lsp.tf` and `windmill_multiplayer.tf`). A plain `terraform apply` of this version migrates such a stack: the `moved` blocks in [load_balancer.tf](./load_balancer.tf) keep the listener rule in place, and the old services and target groups are removed. + +While the new `windmill-extra` task starts (about 2.5 minutes in our test), the `/ws/*` and `/ws_mp/*` routes answer 503, so code intelligence and multiplayer are unavailable in the editor until it is healthy. The debugger (`/ws_debug/*`) stays off in this example unless you set `ENABLE_DEBUGGER=true` in [windmill_extra.tf](./windmill_extra.tf). The Windmill servers and workers are not affected. + ### Ready? REMINDER: don't forget to edit [terraform.tfvars](./terraform.tfvars) before deploying the stack. diff --git a/examples/deploy/aws-ecs-terraform/ecs_cluster.tf b/examples/deploy/aws-ecs-terraform/ecs_cluster.tf index ee5f74d4c4..ddbaad35cf 100644 --- a/examples/deploy/aws-ecs-terraform/ecs_cluster.tf +++ b/examples/deploy/aws-ecs-terraform/ecs_cluster.tf @@ -17,9 +17,10 @@ resource "aws_launch_template" "windmill_cluster_lt" { } network_interfaces { - device_index = 0 - delete_on_termination = true - associate_public_ip_address = true + device_index = 0 + delete_on_termination = true + # Private subnets: outbound traffic goes through the NAT gateways, so no public IP. + associate_public_ip_address = false security_groups = [ aws_security_group.windmill_cluster_sg.id ] @@ -57,6 +58,16 @@ resource "aws_autoscaling_group" "windmill_cluster_asg" { value = true propagate_at_launch = true } + + # The ECS agent on these instances reaches AWS through private route -> NAT gateway -> public + # route -> internet gateway. Keeping that whole path until the instances are gone lets + # `terraform destroy` drain and delete the ECS services, which otherwise hang in DRAINING. + depends_on = [ + aws_route_table_association.windmill_cluster_subnet_private1__rtb_private1, + aws_route_table_association.windmill_cluster_subnet_private2__rtb_private2, + aws_route_table_association.windmill_cluster_subnet_public1__rtb_public, + aws_route_table_association.windmill_cluster_subnet_public2__rtb_public, + ] } resource "aws_ecs_cluster" "windmill_cluster" { diff --git a/examples/deploy/aws-ecs-terraform/load_balancer.tf b/examples/deploy/aws-ecs-terraform/load_balancer.tf index 376374efb6..f73704f04e 100644 --- a/examples/deploy/aws-ecs-terraform/load_balancer.tf +++ b/examples/deploy/aws-ecs-terraform/load_balancer.tf @@ -6,20 +6,20 @@ resource "aws_lb_target_group" "windmill_cluster_windmill_server_tg" { vpc_id = aws_vpc.windmill_cluster_vpc.id } -resource "aws_lb_target_group" "windmill_cluster_windmill_lsp_tg" { - name = "windmill-cluster-lsp-tg" - port = 3001 +resource "aws_lb_target_group" "windmill_cluster_windmill_extra_tg" { + # A prefix, not a fixed name: with create_before_destroy the replacement exists alongside the old + # group for a moment, and target group names must be unique. + name_prefix = "wmext-" + port = 3000 protocol = "HTTP" target_type = "ip" vpc_id = aws_vpc.windmill_cluster_vpc.id -} -resource "aws_lb_target_group" "windmill_cluster_windmill_multiplayer_tg" { - name = "windmill-cluster-multiplayer-tg" - port = 3002 - protocol = "HTTP" - target_type = "ip" - vpc_id = aws_vpc.windmill_cluster_vpc.id + # Replacing a target group that a listener rule uses: create the new one and repoint the rule + # before deleting the old one, which the ALB refuses while the rule still references it. + lifecycle { + create_before_destroy = true + } } resource "aws_lb" "windmill_cluster_alb" { @@ -48,34 +48,32 @@ resource "aws_lb_listener" "windmill_cluster_alb_listener" { } } -resource "aws_lb_listener_rule" "windmill_cluster_alb_lsp_rule" { +resource "aws_lb_listener_rule" "windmill_cluster_alb_extra_rule" { listener_arn = aws_lb_listener.windmill_cluster_alb_listener.arn priority = 100 action { type = "forward" - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_lsp_tg.arn + target_group_arn = aws_lb_target_group.windmill_cluster_windmill_extra_tg.arn } condition { path_pattern { - values = ["/ws/*"] + values = ["/ws/*", "/ws_mp/*", "/ws_debug/*"] } } } -resource "aws_lb_listener_rule" "windmill_cluster_alb_multiplayer_rule" { - listener_arn = aws_lb_listener.windmill_cluster_alb_listener.arn - priority = 50 +# Upgrading a stack created before windmill-extra (standalone LSP and multiplayer services). +# The LSP rule becomes the extra rule, updated in place: it keeps priority 100, where destroying it and +# creating a new rule with the same priority can fail with PriorityInUse. The LSP target group becomes +# the extra target group, replaced before destroy (see create_before_destroy above). +moved { + from = aws_lb_listener_rule.windmill_cluster_alb_lsp_rule + to = aws_lb_listener_rule.windmill_cluster_alb_extra_rule +} - action { - type = "forward" - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_multiplayer_tg.arn - } - - condition { - path_pattern { - values = ["/ws_mp/*"] - } - } +moved { + from = aws_lb_target_group.windmill_cluster_windmill_lsp_tg + to = aws_lb_target_group.windmill_cluster_windmill_extra_tg } diff --git a/examples/deploy/aws-ecs-terraform/rds.tf b/examples/deploy/aws-ecs-terraform/rds.tf index 610f4a61d2..7cceec7494 100644 --- a/examples/deploy/aws-ecs-terraform/rds.tf +++ b/examples/deploy/aws-ecs-terraform/rds.tf @@ -19,7 +19,7 @@ resource "aws_db_instance" "windmill_cluster_rds" { # iops = 3000 engine = "postgres" - engine_version = "16.1" + engine_version = "16" parameter_group_name = "default.postgres16" license_model = "postgresql-license" @@ -39,4 +39,11 @@ resource "aws_db_instance" "windmill_cluster_rds" { backup_retention_period = 7 skip_final_snapshot = true deletion_protection = false + + # engine_version picks the major version when the database is created; RDS then applies minor + # upgrades itself. Changing it on an existing instance would ask RDS to "upgrade" to the bare major + # version, which it rejects, so Terraform leaves it alone after creation. + lifecycle { + ignore_changes = [engine_version] + } } \ No newline at end of file diff --git a/examples/deploy/aws-ecs-terraform/vpc.tf b/examples/deploy/aws-ecs-terraform/vpc.tf index 6e9c66c6bf..79f0a815a1 100644 --- a/examples/deploy/aws-ecs-terraform/vpc.tf +++ b/examples/deploy/aws-ecs-terraform/vpc.tf @@ -69,6 +69,9 @@ resource "aws_nat_gateway" "windmill_cluster_nat_gateway_public1" { tags = { "Name" = "windmill-cluster-nat-gateway-public1" } + + # Created after, and deleted before, the internet gateway it routes through. + depends_on = [aws_internet_gateway.windmill_cluster_internet_gateway] } resource "aws_eip" "windmill_cluster_nat_gateway_public2_eip" { @@ -81,6 +84,9 @@ resource "aws_nat_gateway" "windmill_cluster_nat_gateway_public2" { tags = { "Name" = "windmill-cluster-nat-gateway-public2" } + + # Created after, and deleted before, the internet gateway it routes through. + depends_on = [aws_internet_gateway.windmill_cluster_internet_gateway] } resource "aws_route_table" "windmill_cluster_rtb_public" { diff --git a/examples/deploy/aws-ecs-terraform/windmill_extra.tf b/examples/deploy/aws-ecs-terraform/windmill_extra.tf new file mode 100644 index 0000000000..ecb4de052d --- /dev/null +++ b/examples/deploy/aws-ecs-terraform/windmill_extra.tf @@ -0,0 +1,126 @@ +# windmill-extra: LSP, Multiplayer and Debugger in one image, behind a gateway on port 3000 +# that routes /ws/* (LSP), /ws_mp/* (Multiplayer) and /ws_debug/* (Debugger). +resource "aws_cloudwatch_log_group" "windmill_cluster_windmill_extra_log_group" { + name = "/ecs/windmill-extra" +} + +resource "aws_ecs_task_definition" "windmill_cluster_windmill_extra_td" { + family = "windmill-extra" + network_mode = "awsvpc" + execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn + cpu = 1024 + memory = 1536 + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "X86_64" + } + requires_compatibilities = ["EC2"] + + container_definitions = jsonencode([ + { + name = "windmill-extra" + image = "ghcr.io/windmill-labs/windmill-extra:latest" + cpu = 1024 + memory = 1536 + essential = true + portMappings = [ + { + name = "http" + containerPort = 3000 + hostPort = 3000 + protocol = "tcp" + appProtocol = "http" + } + ] + environment = [ + { + name = "JSON_FMT" + value = "true" + }, + { + # The target group health check (GET /) is answered by the LSP: keep it enabled, or set a + # health_check path on windmill_cluster_windmill_extra_tg that another enabled service answers. + name = "ENABLE_LSP" + value = "true" + }, + { + # Real-time collaboration, Enterprise Edition only. + name = "ENABLE_MULTIPLAYER" + value = "true" + }, + { + # Keep REQUIRE_SIGNED_DEBUG_REQUESTS=true on any internet-reachable deployment. + name = "ENABLE_DEBUGGER" + value = "false" + }, + { + name = "REQUIRE_SIGNED_DEBUG_REQUESTS" + value = "true" + }, + { + # Multiplayer and the debugger verify the tokens the Windmill server signs, with the key + # they fetch from /api/debug/jwks. Without it, every multiplayer + # session is rejected. Use https:// if you add a TLS listener. + name = "WINDMILL_BASE_URL" + value = "http://${aws_lb.windmill_cluster_alb.dns_name}" + }, + ] + mountPoints = [ + { + sourceVolume = "lsp_cache" + containerPath = "/pyls/.cache" + } + ] + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.windmill_cluster_windmill_extra_log_group.name + "awslogs-region" = data.aws_region.current.name + "awslogs-stream-prefix" = "ecs" + } + } + } + ]) + + volume { + name = "lsp_cache" + host_path = "/pyls/.cache" + } +} + +resource "aws_ecs_service" "windmill_cluster_windmill_extra_service" { + name = "windmill-extra" + cluster = aws_ecs_cluster.windmill_cluster.id + task_definition = aws_ecs_task_definition.windmill_cluster_windmill_extra_td.arn + desired_count = 1 + + network_configuration { + subnets = [ + aws_subnet.windmill_cluster_subnet_private1.id, + aws_subnet.windmill_cluster_subnet_private2.id, + ] + security_groups = [aws_security_group.windmill_cluster_sg.id] + } + + force_new_deployment = true + placement_constraints { + type = "distinctInstance" + } + + capacity_provider_strategy { + capacity_provider = aws_ecs_capacity_provider.windmill_cluster_capacity_provider.name + weight = 100 + } + + load_balancer { + target_group_arn = aws_lb_target_group.windmill_cluster_windmill_extra_tg.arn + container_name = "windmill-extra" + container_port = 3000 + } + + # ECS rejects a service whose target group is not yet attached to the load balancer. + depends_on = [ + aws_autoscaling_group.windmill_cluster_asg, + aws_lb_listener_rule.windmill_cluster_alb_extra_rule, + ] +} diff --git a/examples/deploy/aws-ecs-terraform/windmill_lsp.tf b/examples/deploy/aws-ecs-terraform/windmill_lsp.tf deleted file mode 100644 index 02bd2c5d56..0000000000 --- a/examples/deploy/aws-ecs-terraform/windmill_lsp.tf +++ /dev/null @@ -1,85 +0,0 @@ -resource "aws_cloudwatch_log_group" "windmill_cluster_windmill_lsp_log_group" { - name = "/ecs/windmill-lsp" -} - -resource "aws_ecs_task_definition" "windmill_cluster_windmill_lsp_td" { - family = "windmill-lsp" - network_mode = "awsvpc" - execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn - cpu = 1024 - memory = 1536 - runtime_platform { - operating_system_family = "LINUX" - cpu_architecture = "X86_64" - } - requires_compatibilities = ["EC2"] - - container_definitions = jsonencode([ - { - name = "windmill-lsp" - image = "ghcr.io/windmill-labs/windmill-lsp:latest" - cpu = 1024 - memory = 1536 - essential = true - portMappings = [ - { - name = "http" - containerPort = 3001 - hostPort = 3001 - protocol = "tcp" - appProtocol = "http" - } - ] - environment = [{ - name = "JSON_FMT" - value = "true" - }] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.windmill_cluster_windmill_lsp_log_group.name - "awslogs-region" = data.aws_region.current.name - "awslogs-stream-prefix" = "ecs" - } - } - } - ]) - - volume { - name = "lsp_cache" - host_path = "/pyls/.cache" - } -} - -resource "aws_ecs_service" "windmill_cluster_windmill_lsp_service" { - name = "windmill-lsp" - cluster = aws_ecs_cluster.windmill_cluster.id - task_definition = aws_ecs_task_definition.windmill_cluster_windmill_lsp_td.arn - desired_count = 1 - - network_configuration { - subnets = [ - aws_subnet.windmill_cluster_subnet_private1.id, - aws_subnet.windmill_cluster_subnet_private2.id, - ] - security_groups = [aws_security_group.windmill_cluster_sg.id] - } - - force_new_deployment = true - placement_constraints { - type = "distinctInstance" - } - - capacity_provider_strategy { - capacity_provider = aws_ecs_capacity_provider.windmill_cluster_capacity_provider.name - weight = 100 - } - - load_balancer { - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_lsp_tg.arn - container_name = "windmill-lsp" - container_port = 3001 - } - - depends_on = [aws_autoscaling_group.windmill_cluster_asg] -} diff --git a/examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf b/examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf deleted file mode 100644 index a9ccfa0ee2..0000000000 --- a/examples/deploy/aws-ecs-terraform/windmill_multiplayer.tf +++ /dev/null @@ -1,80 +0,0 @@ -resource "aws_cloudwatch_log_group" "windmill_cluster_windmill_multiplayer_log_group" { - name = "/ecs/windmill-multiplayer" -} - -resource "aws_ecs_task_definition" "windmill_cluster_windmill_multiplayer_td" { - family = "windmill-multiplayer" - network_mode = "awsvpc" - execution_role_arn = data.aws_iam_role.ecs_task_execution_role.arn - cpu = 1024 - memory = 1536 - runtime_platform { - operating_system_family = "LINUX" - cpu_architecture = "X86_64" - } - requires_compatibilities = ["EC2"] - - container_definitions = jsonencode([ - { - name = "windmill-multiplayer" - image = "ghcr.io/windmill-labs/windmill-multiplayer:latest" - cpu = 1024 - memory = 1536 - essential = true - portMappings = [ - { - name = "http" - containerPort = 3002 - hostPort = 3002 - protocol = "tcp" - appProtocol = "http" - } - ] - environment = [{ - name = "JSON_FMT" - value = "true" - }] - logConfiguration = { - logDriver = "awslogs" - options = { - "awslogs-group" = aws_cloudwatch_log_group.windmill_cluster_windmill_multiplayer_log_group.name - "awslogs-region" = data.aws_region.current.name - "awslogs-stream-prefix" = "ecs" - } - } - } - ]) -} - -resource "aws_ecs_service" "windmill_cluster_windmill_multiplayer_service" { - name = "windmill-multiplayer" - cluster = aws_ecs_cluster.windmill_cluster.id - task_definition = aws_ecs_task_definition.windmill_cluster_windmill_multiplayer_td.arn - desired_count = 1 - - network_configuration { - subnets = [ - aws_subnet.windmill_cluster_subnet_private1.id, - aws_subnet.windmill_cluster_subnet_private2.id, - ] - security_groups = [aws_security_group.windmill_cluster_sg.id] - } - - force_new_deployment = true - placement_constraints { - type = "distinctInstance" - } - - capacity_provider_strategy { - capacity_provider = aws_ecs_capacity_provider.windmill_cluster_capacity_provider.name - weight = 100 - } - - load_balancer { - target_group_arn = aws_lb_target_group.windmill_cluster_windmill_multiplayer_tg.arn - container_name = "windmill-multiplayer" - container_port = 3002 - } - - depends_on = [aws_autoscaling_group.windmill_cluster_asg] -} diff --git a/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf b/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf index 1c7ff33308..f033d31057 100644 --- a/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf +++ b/examples/deploy/aws-ecs-terraform/windmill_worker_high_performance.tf @@ -17,9 +17,10 @@ resource "aws_launch_template" "windmill_cluster_high_performance_lt" { } network_interfaces { - device_index = 0 - delete_on_termination = true - associate_public_ip_address = true + device_index = 0 + delete_on_termination = true + # Private subnets: outbound traffic goes through the NAT gateways, so no public IP. + associate_public_ip_address = false security_groups = [ aws_security_group.windmill_cluster_sg.id ] @@ -57,6 +58,16 @@ resource "aws_autoscaling_group" "windmill_cluster_high_performance_asg" { value = true propagate_at_launch = true } + + # The ECS agent on these instances reaches AWS through private route -> NAT gateway -> public + # route -> internet gateway. Keeping that whole path until the instances are gone lets + # `terraform destroy` drain and delete the ECS services, which otherwise hang in DRAINING. + depends_on = [ + aws_route_table_association.windmill_cluster_subnet_private1__rtb_private1, + aws_route_table_association.windmill_cluster_subnet_private2__rtb_private2, + aws_route_table_association.windmill_cluster_subnet_public1__rtb_public, + aws_route_table_association.windmill_cluster_subnet_public2__rtb_public, + ] } resource "aws_ecs_capacity_provider" "windmill_cluster_high_performance_capacity_provider" { diff --git a/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml b/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml index 5f2b4c42dc..c26916aa1e 100644 --- a/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml +++ b/examples/deploy/aws-eks-cloudformation/quicklaunch.yaml @@ -436,7 +436,7 @@ Resources: --set windmill.baseDomain=windmill.local \ --set windmill.baseProtocol=http \ --set windmill.appReplicas=${WorkerReplicas} \ - --set windmill.lspReplicas=2 \ + --set windmill.extraReplicas=2 \ --set windmill.workerGroups[0].name=default \ --set windmill.workerGroups[0].mode=worker \ --set windmill.workerGroups[0].replicas=${WorkerReplicas} \ diff --git a/examples/deploy/otel-tracing-grafana/docker-compose.yml b/examples/deploy/otel-tracing-grafana/docker-compose.yml index 525010efad..d75caa0959 100644 --- a/examples/deploy/otel-tracing-grafana/docker-compose.yml +++ b/examples/deploy/otel-tracing-grafana/docker-compose.yml @@ -136,29 +136,34 @@ services: - windmill_index:/tmp/windmill/search - worker_logs:/tmp/windmill/logs - lsp: - image: ghcr.io/windmill-labs/windmill-lsp:latest + # Combined extra services behind one gateway on port 3000: LSP, Multiplayer and Debugger. + # Caddy routes /ws/*, /ws_mp/* and /ws_debug/* here (see the Caddyfile at the repo root). + # - ENABLE_LSP=true (default) - Language Server Protocol for code intelligence + # - ENABLE_MULTIPLAYER=false - Real-time collaboration (Enterprise Edition) + # - ENABLE_DEBUGGER=false - Interactive debugging via DAP WebSocket + windmill_extra: + image: ghcr.io/windmill-labs/windmill-extra:latest pull_policy: always restart: unless-stopped expose: - - 3001 + - 3000 + environment: + - ENABLE_LSP=true + - ENABLE_MULTIPLAYER=false # Set to true to enable multiplayer (Enterprise Edition) + - ENABLE_DEBUGGER=false # Set to true to enable the debugger + - REQUIRE_SIGNED_DEBUG_REQUESTS=true # Do NOT set to false on any internet-reachable deployment + - WINDMILL_BASE_URL=http://windmill_server:8000 volumes: - - lsp_cache:/root/.cache - - multiplayer: - image: ghcr.io/windmill-labs/windmill-multiplayer:latest - deploy: - replicas: 0 # Set to 1 to enable multiplayer, only available on Enterprise Edition - restart: unless-stopped - expose: - - 3002 + - lsp_cache:/pyls/.cache caddy: - image: ghcr.io/windmill-labs/caddy-l4:latest + # Uses the Caddyfile at the repo root. The two are version-coupled: keep this tag in step with + # the caddy image in the root docker-compose.yml. + image: ghcr.io/windmill-labs/caddy-l4:2.11.4-1 restart: unless-stopped # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed volumes: - - ./Caddyfile:/etc/caddy/Caddyfile + - ../../../Caddyfile:/etc/caddy/Caddyfile # - ./certs:/certs # Provide custom certificate files like cert.pem and key.pem to enable HTTPS - See the corresponding section in the Caddyfile ports: # To change the exposed port, simply change 80:80 to :80. No other changes needed diff --git a/examples/deploy/otel-tracing-jaeger/docker-compose.yml b/examples/deploy/otel-tracing-jaeger/docker-compose.yml index 3b186b45fb..b8db8af57e 100644 --- a/examples/deploy/otel-tracing-jaeger/docker-compose.yml +++ b/examples/deploy/otel-tracing-jaeger/docker-compose.yml @@ -136,29 +136,34 @@ services: - windmill_index:/tmp/windmill/search - worker_logs:/tmp/windmill/logs - lsp: - image: ghcr.io/windmill-labs/windmill-lsp:latest + # Combined extra services behind one gateway on port 3000: LSP, Multiplayer and Debugger. + # Caddy routes /ws/*, /ws_mp/* and /ws_debug/* here (see the Caddyfile at the repo root). + # - ENABLE_LSP=true (default) - Language Server Protocol for code intelligence + # - ENABLE_MULTIPLAYER=false - Real-time collaboration (Enterprise Edition) + # - ENABLE_DEBUGGER=false - Interactive debugging via DAP WebSocket + windmill_extra: + image: ghcr.io/windmill-labs/windmill-extra:latest pull_policy: always restart: unless-stopped expose: - - 3001 + - 3000 + environment: + - ENABLE_LSP=true + - ENABLE_MULTIPLAYER=false # Set to true to enable multiplayer (Enterprise Edition) + - ENABLE_DEBUGGER=false # Set to true to enable the debugger + - REQUIRE_SIGNED_DEBUG_REQUESTS=true # Do NOT set to false on any internet-reachable deployment + - WINDMILL_BASE_URL=http://windmill_server:8000 volumes: - - lsp_cache:/root/.cache - - multiplayer: - image: ghcr.io/windmill-labs/windmill-multiplayer:latest - deploy: - replicas: 0 # Set to 1 to enable multiplayer, only available on Enterprise Edition - restart: unless-stopped - expose: - - 3002 + - lsp_cache:/pyls/.cache caddy: - image: ghcr.io/windmill-labs/caddy-l4:latest + # Uses the Caddyfile at the repo root. The two are version-coupled: keep this tag in step with + # the caddy image in the root docker-compose.yml. + image: ghcr.io/windmill-labs/caddy-l4:2.11.4-1 restart: unless-stopped # Configure the mounted Caddyfile and the exposed ports or use another reverse proxy if needed volumes: - - ./Caddyfile:/etc/caddy/Caddyfile + - ../../../Caddyfile:/etc/caddy/Caddyfile # - ./certs:/certs # Provide custom certificate files like cert.pem and key.pem to enable HTTPS - See the corresponding section in the Caddyfile ports: # To change the exposed port, simply change 80:80 to :80. No other changes needed diff --git a/frontend/README_DEV.md b/frontend/README_DEV.md index 9848f333ae..68aa3d7bc9 100644 --- a/frontend/README_DEV.md +++ b/frontend/README_DEV.md @@ -90,6 +90,24 @@ You can configure another proxy to use like so: REMOTE=http://127.0.0.1:8000 REMOTE_LSP=http://127.0.0.1:3001 npm run dev ``` +`REMOTE_LSP` needs a language server on port 3001. The `windmill-extra` image serves one: + +```bash +docker run --rm -p 3001:3001 ghcr.io/windmill-labs/windmill-extra:latest +``` + +To proxy multiplayer and the debugger as well, publish the image's gateway on port 3000 and use `REMOTE_EXTRA`, which +covers `/ws/*`, `/ws_mp/*` and `/ws_debug/*`. Those two services verify tokens signed by your backend, so give the +container its URL (on Linux, add `--add-host=host.docker.internal:host-gateway`): + +```bash +docker run --rm -p 3000:3000 -e WINDMILL_BASE_URL=http://host.docker.internal:8000 ghcr.io/windmill-labs/windmill-extra:latest +REMOTE=http://127.0.0.1:8000 REMOTE_EXTRA=http://127.0.0.1:3000 npm run dev +``` + +To try local changes to `lsp/pyls_launcher.py` or `lsp/Pipfile`, build that image from the repo root with +`docker build -f docker/DockerfileExtra -t windmill-extra-dev .` and run `windmill-extra-dev` instead. + ### Run dev servers on demand A dev server costs 1.1-1.7 GB resident once a page has been browsed, which adds up when diff --git a/lsp/Dockerfile b/lsp/Dockerfile deleted file mode 100644 index b9bba17df2..0000000000 --- a/lsp/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM python:3.12-slim as python-base -FROM node:22-slim as node-base - -FROM python-base -COPY --from=node-base /usr/local /usr/local - -ENV PATH="/usr/local/bin:${PATH}" -ENV PIPENV_VENV_IN_PROJECT=1 -ENV XDG_CACHE_HOME=/pyls/.cache - -RUN apt-get update \ - && apt-get install -y shellcheck wget git ca-certificates \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && pip install pipenv - -RUN npm install -g diagnostic-languageserver pyright - -RUN set -eux; \ - arch="$(dpkg --print-architecture)"; arch="${arch##*-}"; \ - url=; \ - case "$arch" in \ - 'amd64') \ - targz='go1.26.0.linux-amd64.tar.gz'; \ - ;; \ - 'arm64') \ - targz='go1.26.0.linux-arm64.tar.gz'; \ - ;; \ - 'armhf') \ - targz='go1.26.0.linux-armv6l.tar.gz'; \ - ;; \ - *) echo >&2 "error: unsupported architecture '$arch' (likely packaging update needed)"; exit 1 ;; \ - esac; \ - wget "https://golang.org/dl/$targz" -nv && tar -C /usr/local -xzf "$targz" && rm "$targz"; - -ENV PATH="${PATH}:/usr/local/go/bin" -ENV GOBIN=/usr/local/go/bin -RUN /usr/local/go/bin/go install -v golang.org/x/tools/gopls@latest - -RUN pip3 install tornado python-lsp-jsonrpc ruff==0.16.0 - -COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno - -RUN mkdir -p /pyls/.cache - -WORKDIR /pyls -COPY Pipfile . -RUN cat Pipfile -RUN pip install Cython -RUN pipenv install - -COPY pyls_launcher.py . - -RUN mkdir -p /tmp/monaco && chmod -R 777 /tmp/monaco -RUN cd /tmp/monaco && npm install --save-dev windmill-client - -RUN chmod -R a+rX /usr/local && \ - chmod -R a+rX /pyls - -EXPOSE 3001 - -CMD ["sh", "-c", "if [ -n \"$NETRC\" ]; then echo \"$NETRC\" > /root/.netrc && chmod 600 /root/.netrc; fi && if [ -d /root/.cache ]; then export XDG_CACHE_HOME=/root/.cache && cp -r /pyls/.cache /root/.cache; fi && python3 pyls_launcher.py"] diff --git a/lsp/dev.nu b/lsp/dev.nu deleted file mode 100755 index 0beac3cfc1..0000000000 --- a/lsp/dev.nu +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env nu -let tag = $"dev-lsp-(git branch --show-current)"; - -# Build docker image and start server on localhost:3001 -def main [ - --podman(-p) # Use podman - # TODO: - --detach(-d) # Run container in background and print container ID - ] { - if $podman { - podman build --tag $tag .; podman run -p 3001:3001 $tag - } else { - docker build --tag $tag .; docker run -p 3001:3001 $tag - } -} - -# Stop podman container -def "main stop" [] { - try { docker stop (docker ps -q --filter $"ancestor=($tag)") } - try { podman stop (podman ps -q --filter $"ancestor=($tag)") } -} From d3d539291716baf5807a724d60d338cd3c4fe521 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 25 Sep 2026 17:40:13 +0200 Subject: [PATCH 31/91] feat: add an options field to the postgresql resource (#11223) * feat: add an options field to the postgresql resource Co-Authored-By: Claude Opus 5 * fix: keep a literal plus in postgres connection string parameters Co-Authored-By: Claude Opus 5 * fix: pass postgres options to trigger connections Co-Authored-By: Claude Opus 5 * fix: read DATABASE_URL options the way sqlx does Co-Authored-By: Claude Opus 5 * feat: include postgres options in databaseUrlFromResource Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- .../windmill-api-workspaces/src/workspaces.rs | 6 +++ backend/windmill-common/src/lib.rs | 46 +++++++++++++++++-- backend/windmill-trigger-postgres/src/lib.rs | 6 +++ .../windmill-worker/src/duckdb_executor.rs | 26 ++++++++++- .../src/lib/components/ApiConnectForm.svelte | 3 +- .../AddDataTableWizard.svelte | 11 +++++ .../workspaceSettings/addDataTableModel.ts | 1 + .../utils/postgresConnectionString.test.ts | 17 +++++-- .../src/lib/utils/postgresConnectionString.ts | 24 +++++++--- typescript-client/client.ts | 5 +- 10 files changed, 128 insertions(+), 17 deletions(-) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8ebd4eeb26..a45c02e348 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3235,6 +3235,9 @@ pub(crate) async fn pg_dump_database( if let Some(ref sslmode) = pg_db.sslmode { cmd.env("PGSSLMODE", sslmode); } + if let Some(options) = pg_db.non_empty_options() { + cmd.env("PGOPTIONS", options); + } let output = cmd .output() @@ -3372,6 +3375,9 @@ fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { if let Some(ref sslmode) = pg_db.sslmode { cmd.env("PGSSLMODE", sslmode); } + if let Some(options) = pg_db.non_empty_options() { + cmd.env("PGOPTIONS", options); + } cmd } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 1015948882..63ec44daec 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -857,6 +857,7 @@ ta9ELulniZau8zUAtwqwecxodzl+KO8NYj0a9PGgAM64dMqkRtRA8P4UP350Nag3\n\ accept_invalid_certs: None, use_iam_auth: None, region: None, + options: None, } } @@ -944,6 +945,26 @@ ta9ELulniZau8zUAtwqwecxodzl+KO8NYj0a9PGgAM64dMqkRtRA8P4UP350Nag3\n\ assert!(pg(None, None).to_uri().contains("sslmode=prefer")); } + #[test] + fn options_survive_to_uri() { + let mut db = pg(Some("require"), None); + db.options = Some("endpoint=ep-x -c search_path=a&b".to_string()); + let uri = db.to_uri(); + let config: tokio_postgres::Config = uri.parse().unwrap(); + assert_eq!(config.get_options(), db.options.as_deref()); + assert_eq!(PgDatabase::parse_uri(&uri).unwrap().options, db.options); + + db.options = Some(String::new()); + assert!(!db.to_uri().contains("options")); + + // Read the way sqlx reads DATABASE_URL: `+` is a space. + let parsed = PgDatabase::parse_uri("postgres://u@h/db?options=-c+search_path%3Dwm"); + assert_eq!( + parsed.unwrap().options.as_deref(), + Some("-c search_path=wm") + ); + } + /// The other paths default a missing login to `postgres`; Entra must not, or the /// server rejects a role the resource never named. #[test] @@ -994,6 +1015,8 @@ pub struct PgDatabase { pub accept_invalid_certs: Option, pub use_iam_auth: Option, pub region: Option, + /// The libpq `options` startup parameter (e.g. `endpoint=` for Neon, `-c search_path=x`). + pub options: Option, } // Wrapper enum to hold either Tls or NoTls connection @@ -1061,8 +1084,12 @@ impl PgDatabase { } else { urlencoding::encode(&self.host).into_owned() }; + let options = match self.non_empty_options() { + Some(o) => format!("&options={}", urlencoding::encode(o)), + None => String::new(), + }; format!( - "postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}", + "postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}{options}", user = urlencoding::encode(self.login_name()), password = urlencoding::encode(&self.password.as_deref().unwrap_or("")), host = host, @@ -1072,6 +1099,10 @@ impl PgDatabase { ) } + pub fn non_empty_options(&self) -> Option<&str> { + self.options.as_deref().filter(|o| !o.is_empty()) + } + pub async fn connect( &self, main_db: Option<&DB>, @@ -1340,6 +1371,9 @@ impl PgDatabase { .password(token) .dbname(&self.dbname) .ssl_mode(tokio_postgres::config::SslMode::Require); + if let Some(options) = self.non_empty_options() { + config.options(options); + } let (client, connection) = tokio::time::timeout( std::time::Duration::from_secs(20), @@ -1372,9 +1406,14 @@ impl PgDatabase { let port = parsed_url.port(); let dbname = parsed_url.path().trim_start_matches('/').to_string(); let mut sslmode = None; + let mut options = None; + // Form decoding (`+` is a space) on purpose: this parses DATABASE_URL, and the instance's + // own sqlx pool reads it the same way, so connections derived from it must agree. for query in parsed_url.query_pairs() { - if query.0 == "sslmode" { - sslmode = Some(query.1.to_string()); + match query.0.as_ref() { + "sslmode" => sslmode = Some(query.1.to_string()), + "options" => options = Some(query.1.to_string()), + _ => {} } } @@ -1393,6 +1432,7 @@ impl PgDatabase { accept_invalid_certs: None, use_iam_auth: None, region: None, + options, }) } } diff --git a/backend/windmill-trigger-postgres/src/lib.rs b/backend/windmill-trigger-postgres/src/lib.rs index b92860662d..15918fca7b 100644 --- a/backend/windmill-trigger-postgres/src/lib.rs +++ b/backend/windmill-trigger-postgres/src/lib.rs @@ -236,6 +236,8 @@ pub struct Postgres { pub sslmode: String, #[serde(default, deserialize_with = "empty_as_none")] pub root_certificate_pem: Option, + #[serde(default, deserialize_with = "empty_as_none")] + pub options: Option, } #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] @@ -349,6 +351,10 @@ pub async fn get_raw_postgres_connection( config.password(&database.password); } + if let Some(options) = &database.options { + config.options(options); + } + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; let client = if let Some(connector) = connector { let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index aecdb738b3..943703472f 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2692,7 +2692,7 @@ fn datatable_secret_name(alias: &str) -> String { } /// ATTACH a datatable's postgres database through a DuckDB TEMPORARY SECRET holding -/// the connection parameters; only sslmode rides in the ATTACH string. +/// the connection parameters; only sslmode and options ride in the ATTACH string. fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result> { let res: PgDatabase = serde_json::from_value(db_resource)?; // Escape single quotes: each field is embedded in a single-quoted DuckDB literal, @@ -2705,6 +2705,15 @@ fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result "require", _ => "prefer", }; + // Nor an options parameter. The value is quoted for libpq's keyword/value syntax first, + // then escaped for the DuckDB literal around it. + let options = match res.non_empty_options() { + Some(o) => esc(&format!( + " options='{}'", + o.replace('\\', "\\\\").replace('\'', "\\'") + )), + None => String::new(), + }; let secret_name = datatable_secret_name(alias_name); Ok(vec![ "INSTALL postgres;".to_string(), @@ -2717,7 +2726,7 @@ fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result' (TYPE postgres, SECRET …)` would reach a database nobody @@ -3976,6 +3985,19 @@ mod tests { } } + #[test] + fn test_pg_secret_attach_statements_options() { + let db_resource = json!({ "host": "h", "dbname": "d", "options": r"-c search_path='a\b'" }); + let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap(); + let secret_name = datatable_secret_name("dt"); + assert_eq!( + stmts[3], + format!( + r"ATTACH 'sslmode=prefer options=''-c search_path=\''a\\b\''''' AS dt (TYPE postgres, SECRET {secret_name});" + ) + ); + } + #[test] fn test_datatable_secret_name_sanitizes_and_disambiguates() { let a = datatable_secret_name("a.b"); diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index 2255817c5e..9aff73d827 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -126,7 +126,8 @@ host: parts.host, port: parts.port || args?.port, dbname: parts.dbname || args?.dbname, - sslmode: parts.sslmode || args?.sslmode + sslmode: parts.sslmode || args?.sslmode, + options: parts.options || args?.options }, null, 2 diff --git a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte index 0300ddf310..5e5719f3b5 100644 --- a/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte +++ b/frontend/src/lib/components/workspaceSettings/AddDataTableWizard.svelte @@ -1455,6 +1455,17 @@ clearable={false} />
    +
    + Options + wiz.own.fields.options ?? '', (v) => setField('options', v)} + inputProps={{ placeholder: 'endpoint=ep-example-123456' }} + /> +

    + Startup options sent to the server, such as the endpoint ID a Neon pooled or private + connection needs. +

    +
    { expect(parsePostgresConnectionString('postgres://u:p%40ss@host/db')?.password).toBe('p@ss') expect(parsePostgresConnectionString('postgres://u%40corp:p@host/db')?.user).toBe('u@corp') }) + + // libpq only percent-decodes the query, so `+` stays a plus: `Etc/GMT+3` is a timezone, + // `Etc/GMT 3` is a failed connection. + it('keeps a literal + in options, as libpq does', () => { + expect( + parsePostgresConnectionString('postgres://u@h/db?options=-c%20timezone%3DEtc/GMT+3')?.options + ).toBe('-c timezone=Etc/GMT+3') + }) }) // The wizard offers the same connection as a string or as fields and switches between them @@ -94,7 +102,8 @@ describe('composePostgresConnectionString', () => { host: 'db.example.com', port: 6543, dbname: 'mydb', - sslmode: 'require' + sslmode: 'require', + options: 'endpoint=ep-x -c search_path=a&b+c' } expect(parsePostgresConnectionString(composePostgresConnectionString(parts))).toEqual(parts) }) @@ -105,10 +114,10 @@ describe('composePostgresConnectionString', () => { // dangerous ones are precisely the ones a hand-written denylist would miss. describe('unsupportedConnectionParam', () => { it('names a parameter that decides where data lands', () => { - expect(unsupportedConnectionParam('postgres://u:p@h/db?options=-csearch_path%3Dtenant')).toBe( - 'options' - ) expect(unsupportedConnectionParam('postgres://u:p@h/db?search_path=tenant')).toBe('search_path') + expect( + unsupportedConnectionParam('postgres://u:p@h/db?options=-csearch_path%3Dtenant') + ).toBeUndefined() }) // Dropping these saves a *weaker* connection than the one pasted. diff --git a/frontend/src/lib/utils/postgresConnectionString.ts b/frontend/src/lib/utils/postgresConnectionString.ts index d684739904..8b00fbd528 100644 --- a/frontend/src/lib/utils/postgresConnectionString.ts +++ b/frontend/src/lib/utils/postgresConnectionString.ts @@ -35,7 +35,10 @@ function paramsOf(connectionString: string): Map { const query = connectionString.split('?').slice(1).join('?') const params = new Map() if (!query) return params - new URLSearchParams(query).forEach((value, name) => params.set(name, value)) + // URLSearchParams reads `+` as a space, where libpq keeps it (`timezone=Etc/GMT+3`). + new URLSearchParams(query.replaceAll('+', '%2B')).forEach((value, name) => + params.set(name, value) + ) return params } @@ -52,6 +55,7 @@ export type PostgresConnectionParts = { port?: number dbname?: string sslmode?: string + options?: string } /** A lone `%` is not an escape, and a password is free to contain one. */ @@ -73,19 +77,22 @@ export function parsePostgresConnectionString( // By parameter name, never by searching the query text: `sslmode=` also occurs inside // another parameter's *value*, and a substring match there reads someone's // `application_name=sslmode=disable` as a request to turn TLS off. - const sslmode = paramsOf(connectionString).get('sslmode') + const params = paramsOf(connectionString) + const sslmode = params.get('sslmode') + const options = params.get('options') return { user: decode(user), password: password ? decode(password) : undefined, host: host.startsWith('[') ? host.slice(1, -1) : host, port: port ? Number(port) : undefined, dbname: dbname ? decode(dbname) : undefined, - sslmode: sslmode || undefined + sslmode: sslmode || undefined, + options: options || undefined } } -/** The only query parameter the `postgresql` resource has a field for. */ -const REPRESENTABLE_PARAMS = ['sslmode'] +/** The query parameters the `postgresql` resource has a field for. */ +const REPRESENTABLE_PARAMS = ['sslmode', 'options'] /** * Parameters that change nothing about what the connection reaches, how it is secured, or how @@ -135,7 +142,12 @@ export function composePostgresConnectionString(parts: PostgresConnectionParts): ? `${encodeURIComponent(parts.user)}:${encodeURIComponent(parts.password)}` : encodeURIComponent(parts.user) const port = parts.port ? `:${parts.port}` : '' - const query = parts.sslmode ? `?sslmode=${parts.sslmode}` : '' + // encodeURIComponent, not URLSearchParams: libpq does not read `+` as a space. + const params = [ + parts.sslmode && `sslmode=${parts.sslmode}`, + parts.options && `options=${encodeURIComponent(parts.options)}` + ].filter(Boolean) + const query = params.length ? `?${params.join('&')}` : '' const dbname = parts.dbname ? encodeURIComponent(parts.dbname) : '' // A bare IPv6 address would put its own colons where the port separator goes. const host = diff --git a/typescript-client/client.ts b/typescript-client/client.ts index dd2123e746..7e84320346 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -837,7 +837,10 @@ export async function setVariable( */ export async function databaseUrlFromResource(path: string): Promise { const resource = await getResource(path); - return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}`; + const options = resource.options + ? `&options=${encodeURIComponent(resource.options)}` + : ""; + return `postgresql://${resource.user}:${resource.password}@${resource.host}:${resource.port}/${resource.dbname}?sslmode=${resource.sslmode}${options}`; } // TODO(gb): need to investigate more how Polars and DuckDB work in TS From 94e4fb1c842057e92f3775eac0826ac7e396d725 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 25 Sep 2026 18:15:08 +0200 Subject: [PATCH 32/91] fix: carry labels when deploying variables, resources and folders (#11222) * fix: carry labels when deploying variables, resources and folders across workspaces Co-Authored-By: Claude Opus 5 * fix: clear a folder's labels in the target when the source has none Co-Authored-By: Claude Opus 5 * test: pin that the frontend deploy adapter carries variable labels Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- cli/test/deploy_labels_unit.test.ts | 84 +++++++++++++++++++ cli/windmill-utils-internal/src/deploy.ts | 8 ++ .../src/lib/utils_workspace_deploy.test.ts | 32 +++++++ frontend/src/lib/utils_workspace_deploy.ts | 42 +++++++--- 4 files changed, 156 insertions(+), 10 deletions(-) create mode 100644 cli/test/deploy_labels_unit.test.ts diff --git a/cli/test/deploy_labels_unit.test.ts b/cli/test/deploy_labels_unit.test.ts new file mode 100644 index 0000000000..2686eb8cc6 --- /dev/null +++ b/cli/test/deploy_labels_unit.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from "bun:test"; +import { deployItem } from "../windmill-utils-internal/src/deploy.ts"; + +// Variables, resources and folders are deployed from a body built field by field rather than +// spread from the source item, so any field left out of it is silently dropped in the target. +test("deployItem: carries labels for variables, resources and folders", async () => { + const labels = ["prod", "billing"]; + const writes: [string, any][] = []; + const provider = (exists: boolean) => + ({ + existsVariable: async () => exists, + existsResource: async () => exists, + existsFolder: async () => exists, + getVariable: async () => ({ value: "v", is_secret: false, labels }), + getResource: async () => ({ + value: {}, + resource_type: "postgresql", + labels, + }), + getFolder: async () => ({ + name: "x", + owners: [], + extra_perms: {}, + labels, + }), + ...Object.fromEntries( + [ + "createVariable", + "updateVariable", + "createResource", + "updateResource", + "createFolder", + "updateFolder", + ].map((fn) => [ + fn, + async (p: any) => void writes.push([fn, p.requestBody]), + ]), + ), + }) as any; + + for (const exists of [false, true]) { + for (const [kind, path] of [ + ["variable", "f/x/v"], + ["resource", "f/x/r"], + ["folder", "f/x"], + ]) { + const result = await deployItem( + provider(exists), + kind as any, + path, + "src", + "dst", + ); + expect(result).toEqual({ success: true }); + } + } + + expect(writes.map(([fn]) => fn).sort()).toEqual([ + "createFolder", + "createResource", + "createVariable", + "updateFolder", + "updateResource", + "updateVariable", + ]); + for (const [, body] of writes) expect(body.labels).toEqual(labels); +}); + +test("deployItem: clears a folder's labels when the source has none", async () => { + let body: any; + await deployItem( + { + existsFolder: async () => true, + // A folder with cleared labels reads back without the field at all. + getFolder: async () => ({ name: "x", owners: [], extra_perms: {} }), + updateFolder: async (p: any) => void (body = p.requestBody), + } as any, + "folder", + "f/x", + "src", + "dst", + ); + expect(body.labels).toEqual([]); +}); diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts index ac0b775e86..0d70e231cf 100644 --- a/cli/windmill-utils-internal/src/deploy.ts +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -611,6 +611,7 @@ export async function deployItem( value: variable.value ?? "", is_secret: variable.is_secret, description: variable.description ?? "", + labels: variable.labels, }, alreadyEncrypted: false, }); @@ -622,6 +623,7 @@ export async function deployItem( value: variable.value ?? "", is_secret: variable.is_secret, description: variable.description ?? "", + labels: variable.labels, }, }); } @@ -638,6 +640,7 @@ export async function deployItem( path, value: resource.value ?? "", description: resource.description ?? "", + labels: resource.labels, }, }); } else { @@ -648,6 +651,7 @@ export async function deployItem( value: resource.value ?? "", resource_type: resource.resource_type, description: resource.description ?? "", + labels: resource.labels, }, }); } @@ -689,6 +693,9 @@ export async function deployItem( owners: folder.owners, extra_perms: folder.extra_perms, summary: folder.summary ?? undefined, + // A folder stores cleared labels as NULL and reads back without the field, so an + // absent value has to be sent as [] or the target keeps labels the source removed. + labels: folder.labels ?? [], }, }); } else { @@ -699,6 +706,7 @@ export async function deployItem( owners: folder.owners, extra_perms: folder.extra_perms, summary: folder.summary ?? undefined, + labels: folder.labels, }, }); } diff --git a/frontend/src/lib/utils_workspace_deploy.test.ts b/frontend/src/lib/utils_workspace_deploy.test.ts index f220fc93b9..dfac557d20 100644 --- a/frontend/src/lib/utils_workspace_deploy.test.ts +++ b/frontend/src/lib/utils_workspace_deploy.test.ts @@ -7,6 +7,7 @@ import { deployPermissionForKinds, kindGatedByDeployRules, checkPathWritePermission, + deployItem, diffActionableInDirection, diffCreatesInTarget, diffRemovesInTarget @@ -20,6 +21,37 @@ vi.mock('$lib/workspaceProtectionRules.svelte', async (importOriginal) => ({ fetchProtectionRulesForWorkspace: async () => rulesets })) +const variableWrites: [string, any][] = [] +vi.mock('$lib/gen', async (importOriginal) => ({ + ...((await importOriginal()) as object), + VariableService: { + existsVariable: async (p: { workspace: string }) => p.workspace === 'has-it', + getVariable: async () => ({ value: 'v', is_secret: false, labels: ['prod'] }), + createVariable: async (p: any) => void variableWrites.push(['create', p.requestBody]), + updateVariable: async (p: any) => void variableWrites.push(['update', p.requestBody]) + } +})) + +// The deploy runs the published shared package, whose variable body has no `labels`: this +// pins that the adapter still carries them until a version that does is installed. +describe('deploying a variable to another workspace', () => { + it('carries its labels on create and on update', async () => { + for (const workspaceTo of ['lacks-it', 'has-it']) { + const result = await deployItem({ + kind: 'variable', + path: 'u/a/v', + workspaceFrom: 'fork', + workspaceTo + }) + expect(result.success).toBe(true) + } + expect(variableWrites).toEqual([ + ['create', expect.objectContaining({ labels: ['prod'] })], + ['update', expect.objectContaining({ labels: ['prod'] })] + ]) + }) +}) + /** The row shape the fork comparison returns for an item the parent has and the * fork does not: one write on the fork side, whatever that write was. */ const parentOnly = { ahead: 1, behind: 0, exists_in_source: true, exists_in_fork: false } diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index d56c1f6541..ba1f293678 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -228,6 +228,18 @@ function makeProvider( if (conflict) conflict.hit = true throw new Error('item already exists in the target workspace') } + // The published package builds variable, resource and folder bodies field by field without + // `labels`, so carry the source item's across from the read `deployItem` does just before the + // write. One provider serves one deploy, so the read always belongs to the write that follows. + let sourceLabels: string[] | undefined + const readLabels = (item: T): T => { + sourceLabels = item.labels + return item + } + const withLabels = >(requestBody: T): T => ({ + ...requestBody, + labels: sourceLabels + }) return { existsFlowByPath: (p) => FlowService.existsFlowByPath(p), existsScriptByPath: (p) => ScriptService.existsScriptByPath(p), @@ -272,21 +284,31 @@ function makeProvider( getPublicSecretOfLatestVersionOfApp: (p) => AppService.getPublicSecretOfLatestVersionOfApp(p), getRawAppData: (p) => AppService.getRawAppData(p), deleteApp: (p) => AppService.deleteApp(p), - getVariable: (p) => VariableService.getVariable(p), - createVariable: (p) => VariableService.createVariable(p), - updateVariable: (p) => VariableService.updateVariable(p), + getVariable: (p) => VariableService.getVariable(p).then(readLabels), + createVariable: (p) => + VariableService.createVariable({ ...p, requestBody: withLabels(p.requestBody) }), + updateVariable: (p) => + VariableService.updateVariable({ ...p, requestBody: withLabels(p.requestBody) }), deleteVariable: (p) => VariableService.deleteVariable(p), - getResource: (p) => ResourceService.getResource(p), - createResource: (p) => ResourceService.createResource(p), - updateResource: (p) => ResourceService.updateResource(p), + getResource: (p) => ResourceService.getResource(p).then(readLabels), + createResource: (p) => + ResourceService.createResource({ ...p, requestBody: withLabels(p.requestBody) }), + updateResource: (p) => + ResourceService.updateResource({ ...p, requestBody: withLabels(p.requestBody) }), deleteResource: (p) => ResourceService.deleteResource(p), getResourceType: (p) => ResourceService.getResourceType(p), createResourceType: (p) => ResourceService.createResourceType(p), updateResourceType: (p) => ResourceService.updateResourceType(p), deleteResourceType: (p) => ResourceService.deleteResourceType(p), - getFolder: (p) => FolderService.getFolder(p), - createFolder: (p) => FolderService.createFolder(p), - updateFolder: (p) => FolderService.updateFolder(p), + getFolder: (p) => FolderService.getFolder(p).then(readLabels), + createFolder: (p) => + FolderService.createFolder({ ...p, requestBody: withLabels(p.requestBody) }), + // A folder with no labels reads back without the field, so clearing them needs an explicit []. + updateFolder: (p) => + FolderService.updateFolder({ + ...p, + requestBody: { ...p.requestBody, labels: sourceLabels ?? [] } + }), deleteFolder: (p) => FolderService.deleteFolder(p), // Triggers existsTriggerByKind: (kind, p) => triggerServiceFor(kind).exists(p), @@ -670,7 +692,7 @@ export type CreateFolderResult = DeployResult & { * principal's existence at item-create time, so the folder would be created and then reject * every deploy into it, including the retry. * - * `default_permissioned_as` and `labels` are carried at all, which the shared folder deploy drops. + * `default_permissioned_as` is carried at all, which the shared folder deploy drops. */ export async function createFolderIfAbsent( name: string, From e14da5c6bc8256bfe1d71ddecb886c3efc994ae4 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 25 Sep 2026 18:19:38 +0200 Subject: [PATCH 33/91] feat(bedrock): add OIDC role assumption as a fourth auth mode (#10936) * feat(bedrock): add OIDC role assumption as a fourth auth mode Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YCVb91fZp3dqM14KRPzoEn * fix(bedrock): gate the OIDC cache correctly and assume the role once per job Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YCVb91fZp3dqM14KRPzoEn * refactor(bedrock): check the OIDC region before minting a token Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YCVb91fZp3dqM14KRPzoEn * fix(bedrock): keep OIDC session names collision-resistant, gate the copy on EE Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YCVb91fZp3dqM14KRPzoEn * fix(bedrock): check the OIDC region before reusing cached credentials Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YCVb91fZp3dqM14KRPzoEn * fix(bedrock): clear assumed-role sessions when AI settings change invalidate_ai_request_cache_for_workspace cleared AI_REQUEST_CACHE only, so a workspace's AI settings edit reset one cache and left the assumed-role sessions keyed on the old config in place until STS expired them. Also name the region requirement in the credentials-check hint, so following it does not land on the OIDC path's region guard. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01YCVb91fZp3dqM14KRPzoEn * chore: update ee-repo-ref to de73db2bacfdc3eaa2e63b1827178bc198d54e5c This commit updates the EE repository reference after PR #770 was merged in windmill-ee-private. Previous ee-repo-ref: c43dab1e69b1cb3f685e6df07bff634dc2a0b734 New ee-repo-ref: de73db2bacfdc3eaa2e63b1827178bc198d54e5c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-ai/Cargo.toml | 2 +- backend/windmill-ai/src/ai_bedrock.rs | 258 ++++++++++++++++++ backend/windmill-ai/src/credentials.rs | 13 + .../windmill-ai/src/providers/anthropic.rs | 1 + .../windmill-ai/src/providers/google_ai.rs | 1 + backend/windmill-ai/src/providers/mod.rs | 1 + backend/windmill-ai/src/proxy.rs | 1 + backend/windmill-ai/src/types.rs | 7 + backend/windmill-api/src/ai.rs | 156 +++++++++++ backend/windmill-worker/src/ai_executor.rs | 32 ++- .../components/BedrockCredentialsCheck.svelte | 6 + 12 files changed, 475 insertions(+), 5 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 06b1b10a9a..f102167077 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -c099af43bad922fd57ed0449ab717ffa047b5546 +de73db2bacfdc3eaa2e63b1827178bc198d54e5c diff --git a/backend/windmill-ai/Cargo.toml b/backend/windmill-ai/Cargo.toml index 98ea87a4e3..b1dea3ae54 100644 --- a/backend/windmill-ai/Cargo.toml +++ b/backend/windmill-ai/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true [features] default = [] -bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "dep:aws-credential-types", "dep:aws-smithy-types", "dep:aws-config"] +bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "dep:aws-credential-types", "dep:aws-smithy-types", "dep:aws-config", "windmill-common/aws_auth"] mcp = ["dep:windmill-mcp"] [lib] diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index b223725af2..a9318a1f3a 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -2,6 +2,7 @@ //! //! This module provides: //! - BedrockClient: SDK wrapper with bearer token and IAM credentials auth +//! - OIDC role assumption: exchange a Windmill OIDC token for temporary IAM keys //! - Message/tool conversion: OpenAI format <-> Bedrock Converse API format //! - Stream event parsing: Extract text/tool deltas from Bedrock stream events //! @@ -291,6 +292,187 @@ impl BedrockClient { } } +// ============================================================================ +// OIDC Role Assumption +// ============================================================================ + +pub use windmill_common::auth::aws::AWS_OIDC_AUDIENCE; + +/// Bedrock credential precedence: a bearer API key wins, then a complete pair of +/// explicit IAM keys, then OIDC role assumption, then the ambient environment. +/// Returns the role to assume only when both higher-priority modes are unset, so +/// a resource can carry a role ARN without it ever overriding keys set on it. +pub fn bedrock_oidc_role_to_assume<'a>( + api_key: Option<&str>, + aws_access_key_id: Option<&str>, + aws_secret_access_key: Option<&str>, + oidc_role_arn: Option<&'a str>, +) -> Option<&'a str> { + let has_bearer_token = api_key.is_some_and(|key| !key.is_empty()); + let has_iam_keys = aws_access_key_id.is_some_and(|key| !key.is_empty()) + && aws_secret_access_key.is_some_and(|key| !key.is_empty()); + + if has_bearer_token || has_iam_keys { + return None; + } + + oidc_role_arn.filter(|arn| !arn.is_empty()) +} + +/// Re-assume the role this long before AWS expires the credentials, so a request +/// that starts on a reused set cannot finish holding dead ones. +pub const ASSUMED_ROLE_REFRESH_MARGIN: std::time::Duration = std::time::Duration::from_secs(120); + +/// Temporary IAM credentials minted by STS `AssumeRoleWithWebIdentity`. +#[derive(Clone, Debug)] +pub struct AssumedRoleCredentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: String, + pub expires_at: std::time::SystemTime, +} + +impl AssumedRoleCredentials { + /// Whether these have enough life left to sign another request. + pub fn is_fresh(&self) -> bool { + self.expires_at > std::time::SystemTime::now() + ASSUMED_ROLE_REFRESH_MARGIN + } +} + +/// Assume the resource's OIDC role for a job, reusing `cached` while it is still +/// fresh, and hand back the temporary keys to sign Bedrock requests with. +/// +/// The agent loop calls this once per iteration, so without the reuse a +/// tool-heavy run would mint an OIDC token and call STS on every model turn. +/// Returns `None` when a higher-priority credential is set, so the caller can +/// call it unconditionally. +pub async fn refresh_bedrock_oidc_credentials<'a>( + credentials: &crate::credentials::ProviderCredentials, + cached: &'a mut Option, + client: &windmill_common::client::AuthedClient, + job_id: &uuid::Uuid, +) -> Result, Error> { + let Some(role_arn) = bedrock_oidc_role_to_assume( + credentials.api_key.as_deref(), + credentials.aws_access_key_id.as_deref(), + credentials.aws_secret_access_key.as_deref(), + credentials.oidc_role_arn.as_deref(), + ) else { + return Ok(None); + }; + + // Ahead of the reuse check, so a request never signs against the empty region + // just because an earlier one had already assumed the role. + let region = bedrock_oidc_region(credentials.region.as_deref())?; + + if !cached + .as_ref() + .is_some_and(AssumedRoleCredentials::is_fresh) + { + // A worker holds no OIDC signing key, so it asks the API to mint the + // token for this job; the session name carries the job into CloudTrail. + let id_token = client + .get_id_token(AWS_OIDC_AUDIENCE) + .await + .map_err(|e| Error::internal_err(format!("Failed to get OIDC token: {}", e)))?; + // The whole UUID: "windmill-ai-job-" plus 32 hex is 48 characters, well + // inside AWS's limit, and a truncated one would make two runs + // indistinguishable in CloudTrail. + let session_name = aws_role_session_name("windmill-ai-job", &job_id.simple().to_string()); + *cached = + Some(assume_role_with_oidc_token(role_arn, region, id_token, &session_name).await?); + } + + Ok(cached.as_ref()) +} + +/// Distinguishing digest appended to a session name that had to be truncated. +const SESSION_NAME_DIGEST_LEN: usize = 8; + +/// Build a role session name AWS accepts: it constrains them to +/// `[\w+=,.@-]{2,64}`, and rejects the whole request otherwise. Disallowed +/// characters are replaced rather than dropped. +/// +/// The name is the only thing carrying the caller into the assumed-role ARN and +/// CloudTrail, so an identity too long to fit keeps a prefix plus a digest of the +/// whole thing: two identities sharing a prefix would otherwise be +/// indistinguishable in exactly the audit trail this feature exists to produce. +pub fn aws_role_session_name(prefix: &str, identity: &str) -> String { + let sanitize = |s: &str| -> String { + s.chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '_' | '+' | '=' | ',' | '.' | '@' | '-') + { + c + } else { + '-' + } + }) + .collect() + }; + + let name = sanitize(&format!("{}-{}", prefix, identity)); + if name.len() <= 64 { + return name; + } + + let digest = &windmill_common::utils::calculate_hash(identity)[..SESSION_NAME_DIGEST_LEN]; + let kept = 64 - SESSION_NAME_DIGEST_LEN - 1; + format!("{}-{}", &name[..kept], digest) +} + +/// The region an OIDC role assumption runs in. +/// +/// The assumption resolves to explicit IAM keys, and the SDK client built from +/// those has no ambient-region fallback the way [`BedrockClient::from_env`] does, +/// so a resource that assumes a role has to name its region. Callers check this +/// before minting the OIDC token, so a misconfigured resource never issues an +/// identity token it is only going to throw away. +pub fn bedrock_oidc_region(region: Option<&str>) -> Result<&str, Error> { + region.filter(|r| !r.is_empty()).ok_or_else(|| { + Error::BadRequest( + "AWS Bedrock resources that assume a role through OIDC must set a region".to_string(), + ) + }) +} + +/// Exchange a Windmill OIDC token for temporary IAM credentials on `role_arn`. +pub async fn assume_role_with_oidc_token( + role_arn: &str, + region: &str, + id_token: String, + session_name: &str, +) -> Result { + use windmill_common::auth::aws::{ + get_assume_role_with_web_identity_fluent_builder, GetAuthenticationOutput, OidcAuth, + }; + + let oidc_auth = OidcAuth { region: Some(region.to_string()), role_arn: role_arn.to_string() }; + + let output = + get_assume_role_with_web_identity_fluent_builder(&oidc_auth, id_token, Some(session_name)) + .await? + .send() + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to assume AWS role {} with Windmill's OIDC token: {}", + role_arn, + format_bedrock_error(&e) + )) + })?; + + let credentials = output.get_credentials()?; + + Ok(AssumedRoleCredentials { + access_key_id: credentials.access_key_id.clone(), + secret_access_key: credentials.secret_access_key.clone(), + session_token: credentials.session_token.clone(), + expires_at: std::time::UNIX_EPOCH + + std::time::Duration::from_secs(credentials.expiration.secs().max(0) as u64), + }) +} + // ============================================================================ // Error Formatting // ============================================================================ @@ -1000,6 +1182,82 @@ mod tests { use aws_sdk_bedrockruntime::types::ReasoningContentBlock; use serde_json::value::RawValue; + #[test] + fn oidc_role_yields_only_to_bearer_and_complete_iam_keys() { + let arn = Some("arn:aws:iam::1:role/bedrock"); + + assert_eq!( + bedrock_oidc_role_to_assume(None, None, None, arn), + arn, + "nothing else set: assume the role rather than fall through to the environment" + ); + assert_eq!( + bedrock_oidc_role_to_assume(Some("key"), None, None, arn), + None + ); + assert_eq!( + bedrock_oidc_role_to_assume(None, Some("id"), Some("secret"), arn), + None + ); + assert_eq!( + bedrock_oidc_role_to_assume(None, Some("id"), None, arn), + arn, + "half a key pair is not usable IAM credentials, so the role still wins" + ); + // Cleared resource fields arrive as empty strings, not as absent ones. + assert_eq!( + bedrock_oidc_role_to_assume(Some(""), Some(""), Some(""), arn), + arn + ); + assert_eq!( + bedrock_oidc_role_to_assume(None, None, None, Some("")), + None + ); + } + + #[test] + fn credentials_stop_being_fresh_before_aws_expires_them() { + let at = |d: std::time::Duration| AssumedRoleCredentials { + access_key_id: String::new(), + secret_access_key: String::new(), + session_token: String::new(), + expires_at: std::time::SystemTime::now() + d, + }; + // Reused only while there is still margin left to finish a request. + assert!(at(ASSUMED_ROLE_REFRESH_MARGIN * 2).is_fresh()); + assert!(!at(ASSUMED_ROLE_REFRESH_MARGIN / 2).is_fresh()); + } + + #[test] + fn role_session_name_stays_within_what_aws_accepts() { + // A whole job UUID fits, so CloudTrail names the exact run. + assert_eq!( + aws_role_session_name("windmill-ai-job", "0a1b2c3d4e5f60718293a4b5c6d7e8f9"), + "windmill-ai-job-0a1b2c3d4e5f60718293a4b5c6d7e8f9" + ); + // Usernames and emails routinely carry characters AWS rejects. + assert_eq!( + aws_role_session_name("windmill-ai-copilot", "ada/lovelace (admin)"), + "windmill-ai-copilot-ada-lovelace--admin-" + ); + + // Identities too long to fit keep a digest, so a shared prefix does not + // collapse two callers into one CloudTrail identity. + let a = aws_role_session_name("windmill-ai-copilot", &format!("{}a", "x".repeat(60))); + let b = aws_role_session_name("windmill-ai-copilot", &format!("{}b", "x".repeat(60))); + assert_ne!(a, b); + for name in [&a, &b] { + assert_eq!(name.len(), 64); + assert!(name.chars().all(|c| c.is_ascii_alphanumeric() + || matches!(c, '_' | '+' | '=' | ',' | '.' | '@' | '-'))); + } + + // Sanitising before truncating keeps every char one byte, so the cap is + // never applied mid-character. + let unicode = aws_role_session_name("windmill-ai-copilot", &"é".repeat(100)); + assert_eq!(unicode.len(), 64); + } + fn text_message(role: &str, content: &str) -> OpenAIMessage { OpenAIMessage { role: role.to_string(), diff --git a/backend/windmill-ai/src/credentials.rs b/backend/windmill-ai/src/credentials.rs index c9502bdbb2..5a779938ed 100644 --- a/backend/windmill-ai/src/credentials.rs +++ b/backend/windmill-ai/src/credentials.rs @@ -19,6 +19,19 @@ pub struct ProviderCredentials { pub aws_access_key_id: Option, pub aws_secret_access_key: Option, pub aws_session_token: Option, + /// AWS IAM role to assume through Windmill's OIDC provider. Only consulted + /// when no bearer key and no explicit IAM keys are set — see + /// [`crate::ai_bedrock::bedrock_oidc_role_to_assume`]. The API proxy + /// exchanges it for temporary keys before it builds the request; the worker + /// exchanges it at the Bedrock call, because only the API can sign an OIDC + /// token. + /// + /// The two paths therefore present different tokens to AWS: the proxy a + /// workspace claim with `sub = "::"`, the worker the job + /// claim `/oidc/token` issues, `sub = "::::::"`. + /// A role trust policy that conditions on `sub` has to admit both, or only + /// one of the copilot and the AI agent step will be able to assume the role. + pub oidc_role_arn: Option, pub platform: AIPlatform, pub custom_headers: HashMap, } diff --git a/backend/windmill-ai/src/providers/anthropic.rs b/backend/windmill-ai/src/providers/anthropic.rs index 0bb7316dcb..d274d91fa2 100644 --- a/backend/windmill-ai/src/providers/anthropic.rs +++ b/backend/windmill-ai/src/providers/anthropic.rs @@ -858,6 +858,7 @@ mod tests { aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform, custom_headers: HashMap::new(), } diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 8320ef5661..ce37e45df7 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -780,6 +780,7 @@ mod tests { aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform, custom_headers: HashMap::new(), } diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index af9ba3ff9f..908f007978 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -184,6 +184,7 @@ mod parity_tests { aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform: kase.platform.clone(), custom_headers: HashMap::new(), } diff --git a/backend/windmill-ai/src/proxy.rs b/backend/windmill-ai/src/proxy.rs index c672aa1d42..3af8f88a1d 100644 --- a/backend/windmill-ai/src/proxy.rs +++ b/backend/windmill-ai/src/proxy.rs @@ -263,6 +263,7 @@ mod tests { aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform: AIPlatform::Standard, custom_headers: HashMap::new(), } diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 12e06a35d8..ac65f637e2 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -245,6 +245,12 @@ pub struct ProviderResource { deserialize_with = "empty_string_as_none" )] pub aws_session_token: Option, + #[serde( + alias = "oidcRoleArn", + default, + deserialize_with = "empty_string_as_none" + )] + pub oidc_role_arn: Option, /// Platform (standard or google_vertex_ai) #[serde(default)] pub platform: AIPlatform, @@ -311,6 +317,7 @@ impl ProviderWithResource { aws_access_key_id: self.resource.aws_access_key_id.clone(), aws_secret_access_key: self.resource.aws_secret_access_key.clone(), aws_session_token: self.resource.aws_session_token.clone(), + oidc_role_arn: self.resource.oidc_role_arn.clone(), platform: self.resource.platform.clone(), custom_headers: self.resource.headers.clone(), }) diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 16b4b17141..3255b629da 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -120,8 +120,23 @@ lazy_static::lazy_static! { } +#[cfg(feature = "bedrock")] +lazy_static::lazy_static! { + /// Temporary IAM credentials from a Bedrock OIDC role assumption, keyed by + /// (workspace, role ARN, user email). The email is the identity the OIDC + /// token is minted for, so unlike AI_REQUEST_CACHE an entry is never handed + /// to a different user — that is what keeps AWS-side attribution per user. + static ref BEDROCK_ASSUMED_ROLE_CACHE: Cache<(String, String, String), windmill_ai::ai_bedrock::AssumedRoleCredentials> = Cache::new(500); +} + pub(crate) fn invalidate_ai_request_cache_for_workspace(workspace_id: &str) { AI_REQUEST_CACHE.retain(|(cached_workspace_id, _), _| cached_workspace_id != workspace_id); + // Dropped alongside the credentials that named the role, so that changing a + // workspace's AI settings does not leave a session assumed under the old ones + // in play until STS expires it. + #[cfg(feature = "bedrock")] + BEDROCK_ASSUMED_ROLE_CACHE + .retain(|(cached_workspace_id, _, _), _| cached_workspace_id != workspace_id); } /// Shared configuration for every outbound AI HTTP client (the pooled @@ -219,6 +234,12 @@ struct AIStandardResource { deserialize_with = "empty_string_as_none" )] aws_session_token: Option, + #[serde( + alias = "oidcRoleArn", + default, + deserialize_with = "empty_string_as_none" + )] + oidc_role_arn: Option, /// Platform (standard or google_vertex_ai) #[serde(default)] platform: AIPlatform, @@ -301,6 +322,11 @@ async fn resolve_provider_credentials( } else { None }; + let oidc_role_arn = if let Some(role_arn) = resource.oidc_role_arn { + Some(resolve_var(role_arn, db, w_id, user_db.as_ref(), authed).await?) + } else { + None + }; Ok(ProviderCredentials { provider: provider.clone(), @@ -313,6 +339,7 @@ async fn resolve_provider_credentials( aws_access_key_id, aws_secret_access_key, aws_session_token, + oidc_role_arn, platform: resource.platform, custom_headers: resource.headers, }) @@ -337,6 +364,7 @@ async fn resolve_provider_credentials( aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform: AIPlatform::Standard, custom_headers: HashMap::new(), }) @@ -385,6 +413,109 @@ async fn get_token_using_oauth( Ok(response.access_token) } +/// Exchange the resource's OIDC role ARN for temporary IAM credentials, in place. +/// +/// Only the API holds the OIDC signing key, so the exchange happens here rather +/// than in the Bedrock handler, which then sees ordinary IAM keys. It runs after +/// the workspace credential cache on purpose: the STS session name names the user +/// making the request, and a cache shared by the whole workspace would hand one +/// user's session to another. +#[cfg(feature = "bedrock")] +async fn resolve_bedrock_oidc_credentials( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + credentials: &mut ProviderCredentials, +) -> Result<()> { + use windmill_ai::ai_bedrock::{aws_role_session_name, bedrock_oidc_role_to_assume}; + + // Only the Bedrock resource type carries a role ARN; leaving the field set on + // any other resource inert keeps a hand-edited one from reaching STS. + if credentials.provider != AIProvider::AWSBedrock { + return Ok(()); + } + + let Some(role_arn) = bedrock_oidc_role_to_assume( + credentials.api_key.as_deref(), + credentials.aws_access_key_id.as_deref(), + credentials.aws_secret_access_key.as_deref(), + credentials.oidc_role_arn.as_deref(), + ) else { + return Ok(()); + }; + + // Ahead of the cache: STS credentials are region-independent, so a resource + // with the same role but no region would otherwise reuse an entry and sign a + // request against the empty region the guard exists to reject. + let region = windmill_ai::ai_bedrock::bedrock_oidc_region(credentials.region.as_deref())?; + + let session_name = aws_role_session_name("windmill-ai-copilot", &authed.username); + // Keyed on the email, the identity the OIDC token is minted for, rather than + // on the session name derived from it: that name is sanitised and length + // capped, so it is a lossy stand-in for the caller. + let cache_key = (w_id.to_string(), role_arn.to_string(), authed.email.clone()); + + let assumed = match BEDROCK_ASSUMED_ROLE_CACHE + .get(&cache_key) + .filter(windmill_ai::ai_bedrock::AssumedRoleCredentials::is_fresh) + { + Some(cached) => cached, + None => { + let assumed = + assume_bedrock_role(db, w_id, authed, role_arn, region, &session_name).await?; + BEDROCK_ASSUMED_ROLE_CACHE.insert(cache_key, assumed.clone()); + assumed + } + }; + + credentials.aws_access_key_id = Some(assumed.access_key_id); + credentials.aws_secret_access_key = Some(assumed.secret_access_key); + credentials.aws_session_token = Some(assumed.session_token); + + Ok(()) +} + +#[cfg(feature = "bedrock")] +async fn assume_bedrock_role( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + role_arn: &str, + region: &str, + session_name: &str, +) -> Result { + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + { + use windmill_common::oidc_oss::{generate_id_token, WorkspaceClaim}; + + let id_token = generate_id_token( + Some(db), + WorkspaceClaim { workspace: w_id.to_string() }, + windmill_ai::ai_bedrock::AWS_OIDC_AUDIENCE, + format!("{}::{}", authed.email, w_id), + Some(authed.email.clone()), + None, + ) + .await?; + + windmill_ai::ai_bedrock::assume_role_with_oidc_token( + role_arn, + region, + id_token.to_string(), + session_name, + ) + .await + } + #[cfg(not(all(feature = "enterprise", feature = "openidconnect", feature = "private")))] + { + let _ = (db, w_id, authed, role_arn, region, session_name); + Err(Error::BadRequest( + "Assuming an AWS role through Windmill's OIDC provider requires an enterprise license" + .to_string(), + )) + } +} + #[derive(Clone, Debug)] pub struct ExpiringProviderCredentials { credentials: ProviderCredentials, @@ -976,6 +1107,7 @@ async fn global_proxy( aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform: AIPlatform::Standard, custom_headers: HashMap::new(), }; @@ -1253,6 +1385,9 @@ async fn proxy( } }; + #[cfg(feature = "bedrock")] + resolve_bedrock_oidc_credentials(&db, &w_id, &authed, &mut credentials).await?; + // Free tier: pin the model and clamp max_tokens server-side before forwarding, // since the request body is otherwise client-controlled. if free_lease.is_some() { @@ -1463,6 +1598,7 @@ mod tests { aws_access_key_id: None, aws_secret_access_key: None, aws_session_token: None, + oidc_role_arn: None, platform: AIPlatform::Standard, custom_headers: HashMap::new(), } @@ -1504,6 +1640,24 @@ mod tests { ("workspace-b".to_string(), AIProvider::OpenAI), ExpiringProviderCredentials::new(sample_provider_credentials(), None), ); + #[cfg(feature = "bedrock")] + let assumed_role_key = { + let key = ( + "workspace-a".to_string(), + "arn:aws:iam::123456789012:role/bedrock".to_string(), + "someone@windmill.dev".to_string(), + ); + BEDROCK_ASSUMED_ROLE_CACHE.insert( + key.clone(), + windmill_ai::ai_bedrock::AssumedRoleCredentials { + access_key_id: "AKIA".to_string(), + secret_access_key: "secret".to_string(), + session_token: "session".to_string(), + expires_at: std::time::SystemTime::now() + std::time::Duration::from_secs(3600), + }, + ); + key + }; invalidate_ai_request_cache_for_workspace("workspace-a"); @@ -1516,6 +1670,8 @@ mod tests { assert!(AI_REQUEST_CACHE .get(&("workspace-b".to_string(), AIProvider::OpenAI)) .is_some()); + #[cfg(feature = "bedrock")] + assert!(BEDROCK_ASSUMED_ROLE_CACHE.get(&assumed_role_key).is_none()); } #[test] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 3c3a26875b..5f2ef1e6c0 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -1419,6 +1419,11 @@ pub async fn run_agent( .map(|m| m.clamp(1, HARD_MAX_AGENT_ITERATIONS)) .unwrap_or(DEFAULT_MAX_AGENT_ITERATIONS); + // Held across iterations so an OIDC role is assumed once for the job and + // re-assumed only near expiry, rather than on every model turn. + #[cfg(feature = "bedrock")] + let mut bedrock_assumed_role: Option = None; + // Main agent loop for i in 0..max_iterations { // Check if parent was canceled — stop iterating but let current tool calls finish @@ -1438,6 +1443,27 @@ pub async fn run_agent( .region .as_deref() .unwrap_or(windmill_ai::ai_providers::USE_ENV_REGION); + // An OIDC role resolves to ordinary IAM keys, so from here the + // call is identical to the explicit-keys mode. + let assumed = windmill_ai::ai_bedrock::refresh_bedrock_oidc_credentials( + &credentials, + &mut bedrock_assumed_role, + client, + &job.id, + ) + .await?; + let (access_key_id, secret_access_key, session_token) = match assumed { + Some(assumed) => ( + Some(assumed.access_key_id.as_str()), + Some(assumed.secret_access_key.as_str()), + Some(assumed.session_token.as_str()), + ), + None => ( + credentials.aws_access_key_id.as_deref(), + credentials.aws_secret_access_key.as_deref(), + credentials.aws_session_token.as_deref(), + ), + }; // Use Bedrock SDK via dedicated query builder windmill_ai::providers::bedrock::BedrockQueryBuilder::default() .execute_request( @@ -1453,9 +1479,9 @@ pub async fn run_agent( client, &job.workspace_id, structured_output_tool_name.as_deref(), - credentials.aws_access_key_id.as_deref(), - credentials.aws_secret_access_key.as_deref(), - credentials.aws_session_token.as_deref(), + access_key_id, + secret_access_key, + session_token, ) .await? } diff --git a/frontend/src/lib/components/BedrockCredentialsCheck.svelte b/frontend/src/lib/components/BedrockCredentialsCheck.svelte index fa39309a50..726c6d8f0a 100644 --- a/frontend/src/lib/components/BedrockCredentialsCheck.svelte +++ b/frontend/src/lib/components/BedrockCredentialsCheck.svelte @@ -1,5 +1,6 @@ - - drawer?.closeDrawer()}> - {#if activeDrawer} - {#key activeDrawer.key} - {#await activeDrawer.promise} -
    - -
    - {:then Module} - {@const Editor = Module.default as EditorComponent} - - {/await} - {/key} - {/if} -     -     - - - +{#if inner} + {#await inner then Inner} + + {/await} +{/if} diff --git a/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawersInner.svelte b/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawersInner.svelte new file mode 100644 index 0000000000..9dd8fe643c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/CreatedResourceActionDrawersInner.svelte @@ -0,0 +1,192 @@ + + + + drawer?.closeDrawer()}> + {#if activeDrawer} + {#key activeDrawer.key} + {#await activeDrawer.promise} +
    + +
    + {:then Module} + {@const Editor = Module.default as EditorComponent} + + {/await} + {/key} + {/if} +     +     + + + diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 1e486c7d41..6df8b68518 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -60,7 +60,7 @@ import { logFeatureUsage, logHubScriptPick } from '$lib/utils/featureUsage' import { forLater } from '$lib/forLater' import { scriptLangToEditorLang } from '$lib/scripts' import { getCurrentModel } from '$lib/aiStore' -import { type editor as meditor } from 'monaco-editor' +import type { editor as meditor } from 'monaco-editor' import { pendingFolderInstructions, type FolderInstructionsContext } from './folderInstructions' // Prettify function for code arguments - extracts and formats code from JSON diff --git a/frontend/src/lib/components/copilot/shared.ts b/frontend/src/lib/components/copilot/shared.ts index ed37052298..1cdfc239ad 100644 --- a/frontend/src/lib/components/copilot/shared.ts +++ b/frontend/src/lib/components/copilot/shared.ts @@ -1,5 +1,5 @@ import { createLongHash } from '$lib/editorLangUtils' -import { type editor as meditor } from 'monaco-editor' +import type { editor as meditor } from 'monaco-editor' export type VisualChange = | { diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index 2e63266a58..5d3de9150b 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -3,7 +3,7 @@ import { Alert, Badge, ButtonType, Skeleton } from '$lib/components/common' import { capitalize } from '$lib/utils' import NoItemFound from '$lib/components/home/NoItemFound.svelte' - import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' + import { appIconMap } from '$lib/components/icons/appIcon.svelte' import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import ListFilters from '$lib/components/home/ListFilters.svelte' import { ScriptService, type HubScriptKind } from '$lib/gen' @@ -205,8 +205,8 @@ >
    - {#if item['app'] in APP_TO_ICON_COMPONENT} - {@const SvelteComponent = APP_TO_ICON_COMPONENT[item['app']]} + {#if appIconMap()?.[item['app']]} + {@const SvelteComponent = appIconMap()![item['app']]} {/if}
    diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 6648c622c0..e1cfa1d972 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -36,7 +36,7 @@ import { createEventDispatcher, getContext, untrack } from 'svelte' import { Skeleton } from '$lib/components/common' import { classNames, createCache } from '$lib/utils' - import { APP_TO_ICON_COMPONENT } from '$lib/components/icons' + import { appIconMap } from '$lib/components/icons/appIcon.svelte' import { listHubIntegrationsShared } from '$lib/components/displayNameLoaders' import { ScriptService, type HubScriptKind } from '$lib/gen' import { Circle, ExternalLink } from 'lucide-svelte' @@ -253,8 +253,8 @@ onClick={() => handlePickScript(item)} >
    - {#if item['app'] in APP_TO_ICON_COMPONENT} - {@const SvelteComponent = APP_TO_ICON_COMPONENT[item['app']]} + {#if appIconMap()?.[item['app']]} + {@const SvelteComponent = appIconMap()![item['app']]} {:else}
    diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 3c870832ee..0f833084c7 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -52,7 +52,6 @@ import TriggersNode from './renderers/nodes/TriggersNode.svelte' import { Alert, Drawer } from '../common' import Button from '../common/button/Button.svelte' - import FlowYamlEditor from '../flows/header/FlowYamlEditor.svelte' import BranchOneEndNode from './renderers/nodes/branchOneEndNode.svelte' import type { TriggerContext } from '../triggers' import SubflowBound from './renderers/nodes/SubflowBound.svelte' @@ -1296,7 +1295,10 @@ {#if insertable} - + + {#await import('../flows/header/FlowYamlEditor.svelte') then FlowYamlEditor} + + {/await} {/if} {#if canUseDiffDrawer} diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index f86ec9d5bf..1d9edc683b 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -41,7 +41,6 @@ import NoItemFound from './NoItemFound.svelte' import WorkspaceEmptyState from './WorkspaceEmptyState.svelte' import HubProjectPickerModal from './HubProjectPickerModal.svelte' - import ImportProjectModal from './ImportProjectModal.svelte' import type { HubProjectPick } from '$lib/hubProject' import ListFilters from './ListFilters.svelte' import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' @@ -1009,6 +1008,11 @@ // the create menu's Import section open the same dialog, and mounting one per entry point // would put two of them on the page at once while the workspace is still empty. let hubPick = $state(undefined) + // Sticky, so the import modal mounts on the first pick and then stays for its close animation. + let hubImportRequested = $state(false) + $effect(() => { + if (hubPick) hubImportRequested = true + }) let hubPickerOpen = $state(false) /** @@ -2077,7 +2081,13 @@ hubPick = project }} /> - (hubPick = undefined)} {onImported} /> + +{#if hubImportRequested} + {#await import('./ImportProjectModal.svelte') then ImportProjectModal} + (hubPick = undefined)} {onImported} /> + {/await} +{/if} diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 2d88d1deb7..387eea15d0 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -247,8 +247,16 @@ function onWindowKeydownCapture(e: KeyboardEvent) { if (e.key !== 'Escape' || !chatHost.loading) return const active = document.activeElement + // Focus parked on the body answers for the panel on screen only: the flow chat keeps + // a panel per conversation mounted, several of which can be loading, and a hidden + // one's listener would otherwise swallow the press (immediate form, below) and stop + // its own turn instead. `inert` marks the panels behind, and is matched as an + // attribute so the browsers without `checkVisibility` read it too. + const hidden = + panelEl?.closest('[inert]') != null || + panelEl?.checkVisibility?.({ visibilityProperty: true }) === false const focusOnChat = - !active || active === document.body || (panelEl?.contains(active) ?? false) + !hidden && (!active || active === document.body || panelEl?.contains(active) === true) // An Escape while a run form is open must not discard what the user typed, so the action // row alone stops the turn — wherever it is mounted, since the preview panel holds the // form outside `panelEl`. Matched by call: two chats can be loading at once, and one's @@ -487,22 +495,30 @@ const flatFiles = Array.from(dt.files ?? []) const topLevelImages = flatFiles.filter(isImageFile) const imageWork: Promise[] = [] + // The composer this drop landed on, held for the whole routing: a panel destroyed + // while a file is being read clears the binding, and the file would then reach no + // composer at all. + const dropped = aiChatInput if (topLevelImages.length > 0) { - imageWork.push(aiChatInput?.addImages(topLevelImages) ?? Promise.resolve()) + imageWork.push(dropped?.addImages(topLevelImages) ?? Promise.resolve()) } // Text-file routing must await handle/entry resolution before it can call // addTextFiles — hold sending across that window (taken BEFORE the first // await) or a send mid-resolution would land the drop on the next message. - const releaseSendHold = aiChatInput?.holdSendForIngestion() + const releaseSendHold = dropped?.holdSendForIngestion() try { - await routeDroppedTextAndFolders(dt, flatFiles) + await routeDroppedTextAndFolders(dt, flatFiles, dropped) } finally { releaseSendHold?.() } await Promise.all(imageWork) } - async function routeDroppedTextAndFolders(dt: DataTransfer, flatFiles: File[]) { + async function routeDroppedTextAndFolders( + dt: DataTransfer, + flatFiles: File[], + input: typeof aiChatInput + ) { if (canUseFsAccess) { // getAsFileSystemHandle calls are kicked off synchronously inside this call. const handles = await handlesFromDataTransfer(dt) @@ -514,7 +530,10 @@ ? flatFiles : await Promise.all(handles.filter(isFileHandle).map((h) => h.getFile())) // Loose files attach to the message, like images. - await attachNonImageFiles(looseFiles.filter((f) => !isImageFile(f))) + await attachNonImageFiles( + looseFiles.filter((f) => !isImageFile(f)), + input + ) // Folders link as a live handle. const dirs = handles.filter(isDirectoryHandle) if (dirs.length > 0 && !canLinkFolders) { @@ -551,7 +570,7 @@ if (canLinkFolders) await handleAddFiles(folderEntries) else sendUserToast('Folders cannot be attached in this chat — drop individual files.', true) } - await attachNonImageFiles(topLevelText) + await attachNonImageFiles(topLevelText, input) } } @@ -563,8 +582,10 @@ input.value = '' // allow re-selecting the same file } - async function attachNonImageFiles(files: File[]) { - await aiChatInput?.addNonImageFiles(files) + /** `input` is the composer the files are for, captured before any await: read off the + * binding afterwards it would be null once the panel has been destroyed. */ + async function attachNonImageFiles(files: File[], input: typeof aiChatInput) { + await input?.addNonImageFiles(files) } async function attachPickedFiles(picked: File[]) { @@ -572,7 +593,7 @@ const others = picked.filter((f) => !isImageFile(f)) // Reserved before the other work is awaited — see onPanelDrop. const imageWork = imageFiles.length > 0 ? aiChatInput?.addImages(imageFiles) : undefined - await attachNonImageFiles(others) + await attachNonImageFiles(others, aiChatInput) await imageWork } diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index 9744085567..6cb398ee44 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -2,7 +2,7 @@ import FlowModuleSchemaMap from '$lib/components/flows/map/FlowModuleSchemaMap.svelte' import { getContext, tick, untrack } from 'svelte' import type { ExtendedOpenFlow, FlowEditorContext } from '$lib/components/flows/types' - import type { InputTransform } from '$lib/gen' + import { ApiError, type InputTransform } from '$lib/gen' import type { FlowAIChatHelpers } from './core' import { chatMemoryId } from '../global/core' import { createInlineScriptSession } from './inlineScriptsUtils' @@ -174,7 +174,19 @@ previewArgs.val = args } // Call the UI test function which opens preview panel - return await onTestFlow?.(conversationId ?? chatMemoryId(flowStore.val.value)) + try { + return await onTestFlow?.(conversationId ?? chatMemoryId(flowStore.val.value)) + } catch (e) { + // The flow chat is still answering in that conversation, and the server takes one + // turn at a time. Said plainly here: the raw refusal names a job the model has + // no use for. + if (e instanceof ApiError && e.status === 409) { + throw new Error( + 'That chat is still answering an earlier message; wait for it to finish before testing again.' + ) + } + throw e + } }, getLintErrors: async (moduleId: string): Promise => { diff --git a/frontend/src/lib/components/flows/conversations/FlowChat.svelte b/frontend/src/lib/components/flows/conversations/FlowChat.svelte index 01e2f60850..04671e061b 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChat.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChat.svelte @@ -1,13 +1,22 @@
    - {#if chat && chatState} + {#if listChat && listState && pool && poolState} {#if !hideSidebar} @@ -136,24 +204,45 @@ -
    - - {#key chat} - - {/key} +
    + + {#each poolState.mounted as key (key)} + {@const panel = pool.get(key)} + {#if panel} + {@const shown = key === shownKey} +
    + + listState?.conversations.find((c) => c.id === id)?.isTest ?? + conversationKinds.get(id)} + {deploymentInProgress} + {additionalInputsSchema} + {flowModules} + {path} + {identity} + {workspace} + {description} + {wideLayout} + {conversationKind} + {subject} + /> +
    + {/if} + {/each}
    {/if}
    diff --git a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte index cad5688d89..f502be90cb 100644 --- a/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowChatInterface.svelte @@ -3,14 +3,14 @@ import { Loader2, MessageCircle, Settings2 } from 'lucide-svelte' import AIChatDisplay from '$lib/components/copilot/chat/AIChatDisplay.svelte' import { setChatViewHost } from '$lib/components/copilot/chat/chatViewHost' - import { FlowChatViewHost } from './flowChatViewHost.svelte' + import type { FlowChatViewHost } from './flowChatViewHost.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import SchemaForm from '$lib/components/SchemaForm.svelte' import GfmMarkdown from '$lib/components/GfmMarkdown.svelte' import { emptyString, type DynamicInput } from '$lib/utils' - import { onDestroy, tick, untrack } from 'svelte' + import { tick, untrack } from 'svelte' import type { Chat } from 'windmill-chat' - import { chatFlowKey } from './flowChatProps' + import { saveFlowChatInputs } from './flowChatProps' import type { FlowModule } from '$lib/gen' import { useWorkspaceStorageConfigured } from '$lib/components/inputTransformEnv.svelte' import { @@ -30,6 +30,14 @@ interface Props { chat: Chat + /** The conversation's host, which outlives this panel. One panel follows one chat for + * its whole life, so a later value of either prop never reaches it. */ + chatHost: FlowChatViewHost + /** The flow inputs the reader chose, shared by every conversation in the flow: FlowChat + * holds them, since each conversation has a panel of its own mounted. */ + inputValues: Record + /** Whether a conversation is a test chat, once the list has said. */ + isTestOf?: (conversationId: string) => boolean | undefined deploymentInProgress?: boolean additionalInputsSchema?: Record /** The flow's modules, read for the AI agent inputs the composer drives: the provider wiring @@ -50,6 +58,9 @@ let { chat, + chatHost: chatHostProp, + inputValues = $bindable(), + isTestOf = undefined, deploymentInProgress = false, additionalInputsSchema, flowModules, @@ -97,12 +108,7 @@ const modelGap = $derived(agentModelGap(modelWiring, subject)) const showModelButton = $derived(showsModelButton(modelWiring)) - // LocalStorage helpers - const STORAGE_KEY_PREFIX = 'windmill_flow_chat_inputs_' - let showInputsModal = $state(false) - // Conversation settings, persisted per flow: what the reader chose, and nothing else. - let inputValues = $state>(loadInputsFromStorage() ?? {}) let modalDraft = $state>({}) /** What the flow's own form would open on. */ @@ -128,31 +134,9 @@ // value, an author's default — is made safe before it reaches the provider. const runInputs = $derived(withoutRejectedEffort(modelWiring, effectiveInputs)) - function getStorageKey(): string { - return `${STORAGE_KEY_PREFIX}${chatFlowKey({ path, identity })}` - } - - function loadInputsFromStorage(): Record | null { - try { - const stored = localStorage.getItem(getStorageKey()) - return stored ? JSON.parse(stored) : null - } catch (e) { - console.error('Failed to load inputs from localStorage:', e) - return null - } - } - - function saveInputsToStorage(values: Record) { - try { - localStorage.setItem(getStorageKey(), JSON.stringify(values)) - } catch (e) { - console.error('Failed to save inputs to localStorage:', e) - } - } - function setInputValue(name: string, value: any) { inputValues = { ...inputValues, [name]: value } - saveInputsToStorage(inputValues) + saveFlowChatInputs({ path, identity }, inputValues) } function handleModalConfirm() { @@ -167,47 +151,48 @@ ) ) inputValues = kept - saveInputsToStorage(inputValues) + saveFlowChatInputs({ path, identity }, inputValues) showInputsModal = false } function openInputsModal() { - modalDraft = { ...effectiveInputs, ...(loadInputsFromStorage() ?? inputValues) } + modalDraft = { ...effectiveInputs, ...inputValues } showInputsModal = true } - // The host follows the chat it was built on for the life of this component: FlowChat - // remounts the interface under `{#key chat}`, so a later value of the prop never reaches it. - const chatHost = new FlowChatViewHost( - untrack(() => chat), - { - additionalInputs: () => (additionalInputsSchema ? { ...runInputs } : undefined), - attachmentsTarget: () => attachmentsTarget, - attachmentsUnavailable: () => - workspaceStorage.current - ? undefined - : 'This workspace has no object storage, so files cannot be attached.', - workspace: () => workspace, - sendDisabled: () => deploymentInProgress || !!modelGap || !!wrongKindReason, - // The model controls only: a retry changes model when the reader did, but replays - // the run's own attachments rather than whatever the composer holds now. - inputsShownInComposer: () => composerOwnedInputs(modelWiring, undefined) - } - ) + // The host belongs to the conversation, not to this panel: the pool keeps it alive so a + // message queued here still goes out once the reader has moved on. What it reads is this + // panel's, set on mount; one panel shows one conversation for its whole life. + const chatHost = untrack(() => chatHostProp) + chatHost.setOptions({ + additionalInputs: () => (additionalInputsSchema ? { ...runInputs } : undefined), + attachmentsTarget: () => attachmentsTarget, + attachmentsUnavailable: () => + workspaceStorage.current + ? undefined + : 'This workspace has no object storage, so files cannot be attached.', + workspace: () => workspace, + sendDisabled: () => deploymentInProgress || !!modelGap || !!wrongKindReason, + // The model controls only: a retry changes model when the reader did, but replays + // the run's own attachments rather than whatever the composer holds now. + inputsShownInComposer: () => composerOwnedInputs(modelWiring, undefined) + }) setChatViewHost(chatHost) + // Read off the host's own conversation: a message queued here goes out after the reader + // has moved on, when the shown conversation may be of the other kind. + const isTest = $derived.by(() => { + const id = chatHost.state.conversationId + return id === undefined ? undefined : isTestOf?.(id) + }) // A chat of the other kind can be read from here but not added to: the server refuses a // preview run into a deployed conversation and the reverse, so the composer says why first. const wrongKindReason = $derived.by(() => { - const { conversationId, conversations } = chatHost.state - const open = conversations.find((c) => c.id === conversationId) - if (open?.isTest === undefined || open.isTest === (conversationKind === 'test')) - return undefined - return open.isTest + if (isTest === undefined || isTest === (conversationKind === 'test')) return undefined + return isTest ? 'This chat was run from the flow editor. Start a new chat to continue here.' : 'This chat belongs to the deployed flow. Start a new chat to test.' }) - onDestroy(() => chatHost.dispose()) // What the Configure-inputs modal asks for: every flow input the composer does not // edit itself. diff --git a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte index d2ea5c5061..8c6a102fef 100644 --- a/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte +++ b/frontend/src/lib/components/flows/conversations/FlowConversationsSidebar.svelte @@ -5,11 +5,13 @@ Plus, Trash2, Pen, + PencilLine, Filter, PanelLeftClose, PanelLeftOpen } from 'lucide-svelte' - import CountBadge from '$lib/components/common/badge/CountBadge.svelte' + import UnreadCountBadge from '$lib/components/common/badge/UnreadCountBadge.svelte' + import SessionStatusDot from '$lib/components/sessions/SessionStatusDot.svelte' import InfiniteList from '$lib/components/InfiniteList.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' @@ -21,11 +23,16 @@ import { twMerge } from 'tailwind-merge' import { fade } from 'svelte/transition' import { tick, untrack } from 'svelte' - import type { Chat, ChatState, Conversation, ConversationKind } from 'windmill-chat' + import type { Chat, Conversation, ConversationKind } from 'windmill-chat' + import type { FlowChatPool, FlowChatPoolState } from './flowChatPool' + import type { ComposerAttachment, FlowChatViewHost } from './flowChatViewHost.svelte' interface Props { - chat: Chat - chatState: ChatState + /** Lists, renames and deletes the flow's conversations; runs no turn itself. */ + listChat: Chat + /** The conversations' own chats: which one is shown, and what each is doing. */ + pool: FlowChatPool + poolState: FlowChatPoolState /** * Which conversations the list holds at first. The editor shows its own test chats, * since testing is what happens there; a deployed flow shows the chats its users @@ -40,7 +47,13 @@ canFilterKind?: boolean } - let { chat, chatState, defaultKind = 'deployed', canFilterKind = false }: Props = $props() + let { + listChat, + pool, + poolState, + defaultKind = 'deployed', + canFilterKind = false + }: Props = $props() let expanded = $state(false) let list = $state(undefined) @@ -58,13 +71,13 @@ let renameDraft = $state('') let renameInput = $state(undefined) - const turnInFlight = $derived( - chatState.status === 'submitted' || chatState.status === 'streaming' + const totalUnread = $derived( + Object.values(poolState.unread).reduce((total, count) => total + count, 0) ) $effect(() => { const l = list - const c = chat + const c = listChat if (!l) return untrack(() => { // Every load goes through here, the first one and infinite scroll included. A @@ -72,13 +85,18 @@ // selected kind brings its own, whichever of the two lands last. l.setLoader(async (page, perPage) => { const requested = kind + // Only a listing read now says which turns run: the rows the chat holds keep + // what the last one said, and a rename or delete publishes those again. + const since = pool.listingStarted() const rows = await c.loadConversations({ page, perPage, kind: requested }) + pool.setListed(rows, since) return requested === kind ? rows : items }) l.setDeleteItemFn(async (id: string) => { deletingId = id try { await c.deleteConversation(id) + pool.forget(id) sendUserToast('Conversation deleted successfully') } catch (error) { console.error('Failed to delete conversation:', error) @@ -103,10 +121,10 @@ await list?.loadData('forceRefresh') } - const draftShown = $derived(draft && !items.some((c) => c.id === chatState.conversationId)) + const draftShown = $derived(draft && poolState.selectedId === undefined) function newChat() { - chat.newConversation() + pool.newChat() draft = true } @@ -118,17 +136,16 @@ /** * Narrow the list to one kind of chat and reload it. The open conversation goes with it - * when it is not of the new kind: the composer sends into whatever is selected, and a + * when it is not of the new kind: the composer sends into whatever is shown, and a * conversation keeps the kind it was created with, so a turn sent into one the list no - * longer shows would be stored where nothing here lists it. + * longer shows would be stored where nothing here lists it. A turn running in it goes on. */ async function setKind(next: ConversationKind) { - // A turn writes into the open conversation, which a kind that excludes it would close. - if (next === kind || turnInFlight) return + if (next === kind) return kind = next - const open = items.find((c) => c.id === chatState.conversationId) + const open = items.find((c) => c.id === poolState.selectedId) const stillListed = open === undefined || next === 'all' || (next === 'test') === open.isTest - if (!stillListed) chat.newConversation() + if (!stillListed) pool.newChat() await list?.loadData('forceRefresh') } @@ -149,11 +166,11 @@ const current = items.find((c) => c.id === id) if (!current || title === '' || title === current.title) return try { - await chat.renameConversation(id, title) + await listChat.renameConversation(id, title) // The list holds its own rows, loaded through the loader: patched rather than // reloaded, so the row keeps its place without a round trip. The title is read // back from the chat, which holds it as the server stored it (a long one is cut). - const stored = chat.getState().conversations.find((c) => c.id === id)?.title ?? title + const stored = listChat.getState().conversations.find((c) => c.id === id)?.title ?? title items = items.map((c) => (c.id === id ? { ...c, title: stored } : c)) } catch (error) { console.error('Failed to rename conversation:', error) @@ -177,8 +194,34 @@ function getConversationTitle(conversation: Conversation): string { return conversation.title || `Conversation ${conversation.createdAt.slice(0, 10)}` } + + /** The session sidebar's dot vocabulary, which knows no `queued`: that is its own mark. */ + function dotStatus(conversationId: string): 'streaming' | 'error' | 'idle' { + const activity = poolState.activity[conversationId] + return activity === 'running' ? 'streaming' : activity === 'error' ? 'error' : 'idle' + } +{#snippet statusDot(conversation: Conversation)} + + + {#snippet resting()} + + {/snippet} + +{/snippet} +
    + {#snippet trigger()} @@ -233,10 +271,7 @@ unifiedSize="md" variant="subtle" startIcon={{ icon: Filter }} - disabled={turnInFlight} - title={turnInFlight - ? 'Wait for the current answer to change which chats are listed' - : `Filter conversations · ${KIND_LABELS[kind]}`} + title={`Filter conversations · ${KIND_LABELS[kind]}`} iconOnly /> {/snippet} @@ -245,7 +280,6 @@ setKind(next as ConversationKind)} - disabled={turnInFlight} noWFull > {#snippet children({ item })} @@ -274,11 +308,19 @@ unifiedSize="md" startIcon={{ icon: MessageCircle }} onClick={() => (expanded = true)} - title="{items.length} conversation{items.length !== 1 ? 's' : ''}" + title="{items.length} conversation{items.length !== 1 ? 's' : ''}{totalUnread > 0 + ? `, ${totalUnread} unread` + : ''}" variant="subtle" btnClasses="w-fit px-2 relative" > - + +
    {/if} @@ -299,7 +341,6 @@ onClick={(e) => { e?.stopPropagation() draft = false - chat.newConversation() }} title="Discard draft" destructive @@ -314,7 +355,7 @@
    {:else} + {@const unread = poolState.unread[conversation.id] ?? 0} + {@const queued = !!poolState.queued[conversation.id]}     diff --git a/frontend/src/lib/components/sessions/SessionStatusDot.svelte b/frontend/src/lib/components/sessions/SessionStatusDot.svelte index 646163bcaf..e97de79caf 100644 --- a/frontend/src/lib/components/sessions/SessionStatusDot.svelte +++ b/frontend/src/lib/components/sessions/SessionStatusDot.svelte @@ -8,8 +8,18 @@ let { status, isFork, - forkDetached = false - }: { status: SessionChatStatus; isFork: boolean; forkDetached?: boolean } = $props() + forkDetached = false, + resting, + restingTitle + }: { + status: SessionChatStatus + isFork: boolean + forkDetached?: boolean + /** What the slot shows when there is no live signal. Sessions leave it unset and get + * the workspace/fork mark below; another list passes its own resting mark. */ + resting?: import('svelte').Snippet + restingTitle?: string + } = $props() const statusTooltip: Record = { idle: 'No chat activity', @@ -39,7 +49,7 @@ : 'Root workspace session' ) - const title = $derived(liveOverride ? statusTooltip[status] : persistentTitle) + const title = $derived(liveOverride ? statusTooltip[status] : (restingTitle ?? persistentTitle)) @@ -55,6 +65,8 @@ {:else if status === 'error'} + {:else if resting} + {@render resting()} {:else if isFork} {#if forkDetached} From 17448c97d33ddd2e2c6b5aef54699a12226e150f Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 29 Sep 2026 12:26:11 +0200 Subject: [PATCH 66/91] count trigger suspend, resume and discard (#11405) * feat(telemetry): count trigger suspend, resume and discard Co-Authored-By: Claude Opus 5.5 * refactor: shorten the telemetry disclosure to one line per category Co-Authored-By: Claude Opus 5.5 * docs: correct the resume branch comments and note the pre-commit fire count Co-Authored-By: Claude Opus 5.5 * fix: name feature adoption in the telemetry disclosure Co-Authored-By: Claude Opus 5.5 * chore: update ee-repo-ref to 9855e1b7a43a0a33e04f8accf1c497af3fd9b139 This commit updates the EE repository reference after PR #834 was merged in windmill-ee-private. Previous ee-repo-ref: 1d5b128ec956c156fe549cf099ba0dbc1b6467bf New ee-repo-ref: 9855e1b7a43a0a33e04f8accf1c497af3fd9b139 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: windmill-internal-app[bot] --- .agents/skills/svelte-frontend/SKILL.md | 4 +- ...3500a39efa20e4915ffa56fd40537597db36e.json | 26 ----- ...a5c6acbf5468c190fa72b6b6707d950267752.json | 32 ++++++ backend/ee-repo-ref.txt | 2 +- backend/windmill-queue/src/jobs.rs | 12 ++- .../windmill-trigger/src/global_handler.rs | 22 +++- docs/feature-telemetry.md | 11 +- .../lib/components/InstanceSettings.svelte | 101 +++--------------- 8 files changed, 88 insertions(+), 122 deletions(-) delete mode 100644 backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json create mode 100644 backend/.sqlx/query-872c06b3098cf59797bcaa90d6da5c6acbf5468c190fa72b6b6707d950267752.json diff --git a/.agents/skills/svelte-frontend/SKILL.md b/.agents/skills/svelte-frontend/SKILL.md index 6aceedc25c..ddeb8e59a1 100644 --- a/.agents/skills/svelte-frontend/SKILL.md +++ b/.agents/skills/svelte-frontend/SKILL.md @@ -119,8 +119,8 @@ Form components (TextInput, Toggle, Select, etc.) should use the unified size sy New user-facing UX is the main source of `feature_usage` counters — propose them in the plan, not as a separate question, and read `docs/feature-telemetry.md` first. `logFeatureUsage()` from `$lib/utils/featureUsage` is only half the change: the `(feature, kind)` pair must also be -registered in the backend allowlist or every event is silently discarded, and the disclosure copy -in `InstanceSettings.svelte` must name what you added. +registered in the backend allowlist or every event is silently discarded. The disclosure copy in +`InstanceSettings.svelte` stays at the category level; see the recipe's step 3. ## Svelte MCP Server diff --git a/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json b/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json deleted file mode 100644 index e8ffcfb95b..0000000000 --- a/backend/.sqlx/query-1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT 'schedule' AS \"kind!\", COUNT(*)::BIGINT AS \"count!\" FROM schedule\n UNION ALL SELECT 'http', COUNT(*)::BIGINT FROM http_trigger\n UNION ALL SELECT 'websocket', COUNT(*)::BIGINT FROM websocket_trigger\n UNION ALL SELECT 'kafka', COUNT(*)::BIGINT FROM kafka_trigger\n UNION ALL SELECT 'nats', COUNT(*)::BIGINT FROM nats_trigger\n UNION ALL SELECT 'postgres', COUNT(*)::BIGINT FROM postgres_trigger\n UNION ALL SELECT 'mqtt', COUNT(*)::BIGINT FROM mqtt_trigger\n UNION ALL SELECT 'sqs', COUNT(*)::BIGINT FROM sqs_trigger\n UNION ALL SELECT 'gcp', COUNT(*)::BIGINT FROM gcp_trigger\n UNION ALL SELECT 'azure', COUNT(*)::BIGINT FROM azure_trigger\n UNION ALL SELECT 'amqp', COUNT(*)::BIGINT FROM amqp_trigger\n UNION ALL SELECT 'email', COUNT(*)::BIGINT FROM email_trigger\n -- Grouped, not a single 'native' key: these fire as nextcloud/google/github,\n -- so a lone key would not line up with the `trigger`/`fired` series.\n UNION ALL SELECT service_name::text, COUNT(*)::BIGINT FROM native_trigger GROUP BY service_name\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "kind!", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "1d346a14ad5586af347b8e7ac413500a39efa20e4915ffa56fd40537597db36e" -} diff --git a/backend/.sqlx/query-872c06b3098cf59797bcaa90d6da5c6acbf5468c190fa72b6b6707d950267752.json b/backend/.sqlx/query-872c06b3098cf59797bcaa90d6da5c6acbf5468c190fa72b6b6707d950267752.json new file mode 100644 index 0000000000..3bdcb5e48c --- /dev/null +++ b/backend/.sqlx/query-872c06b3098cf59797bcaa90d6da5c6acbf5468c190fa72b6b6707d950267752.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT 'schedule' AS \"kind!\", COUNT(*)::BIGINT AS \"count!\", 0::BIGINT AS \"suspended!\" FROM schedule\n UNION ALL SELECT 'http', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM http_trigger\n UNION ALL SELECT 'websocket', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM websocket_trigger\n UNION ALL SELECT 'kafka', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM kafka_trigger\n UNION ALL SELECT 'nats', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM nats_trigger\n UNION ALL SELECT 'postgres', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM postgres_trigger\n UNION ALL SELECT 'mqtt', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM mqtt_trigger\n UNION ALL SELECT 'sqs', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM sqs_trigger\n UNION ALL SELECT 'gcp', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM gcp_trigger\n UNION ALL SELECT 'azure', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM azure_trigger\n UNION ALL SELECT 'amqp', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM amqp_trigger\n UNION ALL SELECT 'email', COUNT(*)::BIGINT, COUNT(*) FILTER (WHERE mode = 'suspended')::BIGINT FROM email_trigger\n -- Grouped, not a single 'native' key: these fire as nextcloud/google/github,\n -- so a lone key would not line up with the `trigger`/`fired` series.\n UNION ALL SELECT service_name::text, COUNT(*)::BIGINT, 0::BIGINT FROM native_trigger GROUP BY service_name\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "suspended!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "872c06b3098cf59797bcaa90d6da5c6acbf5468c190fa72b6b6707d950267752" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 263784f2aa..58f9aaf07d 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -421cf2a8b4f98b421e93c0fc7c1c378314a66e50 +9855e1b7a43a0a33e04f8accf1c497af3fd9b139 diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 880b72cc0e..7b552fb02c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -6869,9 +6869,19 @@ async fn push_inner<'c, 'd>( // `schedule_path` (see `FlowJob::schedule_path`), so counting per push would // score one run as a fire per step job — a loop pushes two of those per // iteration — burying every other kind, and would sit on the per-step path. + // + // A job a suspended trigger parks is not a fire: it counts as `fired` only + // when `resume_suspended_trigger_jobs` releases it, or never if discarded. + // Both are counted before the queue caps below and the caller's commit, so a + // rejected push still counts; accepted for a telemetry counter. if flow_step_id.is_none() { if let Some(kind) = trigger_kind.as_ref() { - windmill_common::feature_usage::log_feature_usage("trigger", "fired", kind.as_str()); + let action = if suspended_mode.unwrap_or(false) { + "suspended" + } else { + "fired" + }; + windmill_common::feature_usage::log_feature_usage("trigger", action, kind.as_str()); } } diff --git a/backend/windmill-trigger/src/global_handler.rs b/backend/windmill-trigger/src/global_handler.rs index 6e1c92d588..ec74b4f139 100644 --- a/backend/windmill-trigger/src/global_handler.rs +++ b/backend/windmill-trigger/src/global_handler.rs @@ -14,6 +14,7 @@ use windmill_api_jobs::execution::cancel_jobs; use windmill_common::{ db::{UserDB, DB}, error::{self, Error, Result}, + feature_usage::log_feature_usage, jobs::{delete_jobs, JobTriggerKind}, triggers::TriggerMetadata, }; @@ -168,14 +169,16 @@ pub async fn resume_suspended_trigger_jobs( .await? }; - let trigger_metadata = TriggerMetadata::new(Some(trigger_path.clone()), trigger_kind); + let trigger_metadata = TriggerMetadata::new(Some(trigger_path.clone()), trigger_kind.clone()); let l = jobs.len(); + let mut unsuspended_in_place = 0; for job in jobs { - // If job was created before trigger was edited, simply update it to unsuspend + // If job was created after trigger was edited, simply update it to unsuspend // instead of deleting and repushing if job.created_at > trigger.edited_at { + unsuspended_in_place += 1; // Map the placeholder unassigned kind back to its assigned counterpart so // singlestepflow wrappers (retry/error_handler/skip_handler) keep their // flow-orchestrator identity. @@ -209,7 +212,7 @@ pub async fn resume_suspended_trigger_jobs( .execute(&mut *tx) .await?; } else { - // Job was created after trigger edit - delete and repush with new configuration + // Job was created before trigger edit - delete and repush with new configuration // Pass the transaction to trigger_runnable_inner so everything is in the same transaction let (_uuid, _delete_after_use, _early_return, _has_failure_module, tx_o) = trigger_runnable_inner( @@ -274,6 +277,16 @@ pub async fn resume_suspended_trigger_jobs( tx.commit().await?; + // A repushed job is counted as `fired` inside `push`, before this commit, so + // a resume that rolls back still counts it; accepted for a telemetry counter. + // One unsuspended in place never goes through `push`, so it is counted here. + for _ in 0..unsuspended_in_place { + log_feature_usage("trigger", "fired", trigger_kind.as_str()); + } + for _ in 0..l { + log_feature_usage("trigger", "resumed", trigger_kind.as_str()); + } + Ok(Json(format!("Reassigned {} jobs", l))) } @@ -334,6 +347,9 @@ pub async fn cancel_suspended_trigger_jobs( true, ) .await?; + for _ in 0..cancelled_jobs.0.len() { + log_feature_usage("trigger", "discarded", trigger_kind.as_str()); + } Ok(Json(format!("Canceled {} jobs", cancelled_jobs.0.len()))) } else { Ok(Json(format!("No jobs to cancel"))) diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index 9dacfc1cbd..c0d7b9c027 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,7 +4,7 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 58 registered actions across twenty features (`ai_session`, `ai_chat`, +It currently carries 61 registered actions across twenty features (`ai_session`, `ai_chat`, `ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `db_manager`, `flow_editor`, `flow_run`, `flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, `usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the @@ -48,7 +48,7 @@ vocabulary closed and small — enumerate the values in a TS union next to the c ## The recipe -Four steps. Skipping step 1 or 3 fails quietly. +Four steps. Skipping step 1 fails quietly. **1. Register the pair** in `FEATURE_USAGE_KINDS` (`backend/windmill-common/src/feature_usage_ee.rs`, tracked in `windmill-ee-private`). An @@ -68,9 +68,10 @@ Fire-and-forget. Events sum locally per `(workspace, feature, kind, key, entityI every 30s, on `visibilitychange` → hidden, and on `pagehide`; 50 events per request, and a failed batch is dropped rather than retried. -**3. Update the disclosure.** `InstanceSettings.svelte` lists what a non-minimal payload contains -(two places — the copy appears twice). A new counter that isn't named there means the instance -under-discloses what it sends. This has already drifted once. +**3. Check the disclosure.** `InstanceSettings.svelte` describes a non-minimal payload by +category, not counter by counter (two places — the copy appears twice); keep it short. A plain +counter needs no edit. Only a key that sends a new kind of value beyond a count — a name or an +identifier, like the AI model and hub names it already lists — must be added there. **4. Verify a row lands.** The silent-drop path means "no error" proves nothing: diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 7cb8ced225..0c27a3c7c4 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1065,54 +1065,19 @@ Telemetry is required on Enterprise Edition for license compliance. When minimal telemetry is enabled, only the following data is sent:
      -
    • version of your instance
      • -
    • instance base URL
      • -
    • login type usage (login type, count)
      • -
    • worker usage (worker, worker instance, vCPUs, memory)
      • -
    • user usage (author count, operator count, the distinct guests of the last 30 days, - the seats they add past the free allowance, and the workspaces that allow guests)
      • -
    • superadmin email addresses
      • -
    • development instance status
      • +
    • Instance version and base URL
      • +
    • Login types, user and seat counts
      • +
    • Worker count, vCPUs and memory
      • +
    • Superadmin email addresses
      • +
    • Development instance status

    When minimal telemetry is disabled, the following is also collected:
      -
    • job usage (language, total duration, count)
      • -
    • git sync repo count (sync vs promotion mode)
      • -
    • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, the languages debug sessions - are started for, whether AI chat skills are turned on or off and how often one is - loaded, whether an AI agent run narrows the tools it may call and whether that leaves - it with none, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and - change a membership, the plan tier and quota shown when the execution meter is opened, - whether app sandbox isolation is turned on, whether a step's workspace script is - edited from the flow editor, which skin approval steps are given, how many AI sessions - are brought back from the workspace object storage backup, how data tables and their - migrations are set up and used, how often the database manager is switched between its - data and schema diagram views, how often an empty workspace home is seen, how often - the home page’s create menu and hub-project picker are opened and from which entry - point, the name of any public hub project imported from the home page and how far that - import got, whether a pre-approved trial offer was opened, whether data tables are put - under roles and whether callers name a role or take the default, which kinds of access - change (grant, revoke, ownership, default privileges) are applied to data tables, and - whether a data table under roles is cloned into a fork with its schema only or with - its data, last 30 days)
      • -
    • feature adoption (counts of which flow, script, trigger, worker and data table - features your deployed items use, including how many apps run sandboxed, how many data - tables exist per database kind, how many use migrations, and what references them)
      • -
    • resource counts (workspaces, scripts per language, flows, workflows as code, low-code - apps, raw apps)
      • -
    • infrastructure info (container runtime, managed database provider, database version, - size and cluster size, max and active connections, object storage backend)
      • +
    • Job counts and durations per language
      • +
    • Git sync repository counts
      • +
    • Feature usage and adoption counts, including AI model and public Hub names
      • +
    • Counts of workspaces, scripts, flows and apps
      • +
    • Database, runtime and object storage details

    For air-gapped instances, you can download the telemetry data and send it manually.
    @@ -1141,45 +1106,13 @@ Anonymous usage data is collected to help improve Windmill.
    The following information is collected:
      -
    • version of your instance
      • -
    • instance base URL
      • -
    • job usage (language, total duration, count)
      • -
    • login type usage (login type, count)
      • -
    • worker usage (worker, worker instance, vCPUs, memory)
      • -
    • user usage (author count, operator count, the distinct guests of the last 30 days, - the seats they add past the free allowance, and the workspaces that allow guests)
      • -
    • development instance status
      • -
    • feature usage (counts of which product features are used, including AI provider and - model identifiers, the names of public hub scripts used, the languages debug sessions - are started for, whether AI chat skills are turned on or off and how often one is - loaded, whether an AI agent run narrows the tools it may call and whether that leaves - it with none, whether SSO logins evaluate an IdP groups claim (SAML or OIDC) and - change a membership, the plan tier and quota shown when the execution meter is opened, - whether app sandbox isolation is turned on, whether a step's workspace script is - edited from the flow editor, which skin approval steps are given, how many AI sessions - are brought back from the workspace object storage backup, how data tables and their - migrations are set up and used, how often the database manager is switched between its - data and schema diagram views, how often an empty workspace home is seen, how often - the home page’s create menu and hub-project picker are opened and from which entry - point, the name of any public hub project imported from the home page and how far that - import got, whether a pre-approved trial offer was opened, whether data tables are put - under roles and whether callers name a role or take the default, which kinds of access - change (grant, revoke, ownership, default privileges) are applied to data tables, and - whether a data table under roles is cloned into a fork with its schema only or with - its data, last 30 days)
      • -
    • feature adoption (counts of which flow, script, trigger, worker and data table - features your deployed items use, including how many apps run sandboxed, how many data - tables exist per database kind, how many use migrations, and what references them)
      • -
    • resource counts (workspaces, scripts per language, flows, workflows as code, low-code - apps, raw apps)
      • +
    • Instance version and base URL
      • +
    • Job counts and durations per language
      • +
    • Login types, user and seat counts
      • +
    • Worker count, vCPUs and memory
      • +
    • Development instance status
      • +
    • Feature usage and adoption counts, including AI model and public Hub names
      • +
    • Counts of workspaces, scripts, flows and apps
    {/if} From 97fa55b719a874067d1a81a1d680a42d1f1a4c98 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 29 Sep 2026 12:40:25 +0200 Subject: [PATCH 67/91] feat: detect and alert when a schedule skips occurrences (#10917) * docs: plan for detecting skipped schedule occurrences Design plan only, no implementation. Records the scheduler's re-anchoring behaviour, the measurements behind it, and the three-piece design that came out of reviewing the alternatives. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * docs: state the user-facing outcome in the schedule plan The plan described the mechanism but never what a user would see, which made it hard to judge what the work is worth. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * docs: state which cause the schedule plan catches, and correct its scope Records which of the two causes each piece covers, and corrects the overrun scope: a script schedule carrying retry or dynamic_skip is pushed as a SingleStepFlow, so it re-arms at step 0 entry and its occurrences overlap like a flow's. Resolves the no_flow_overlap question, splits the read-time work into bounded detection and editor-only counting behind measured croner costs, and fixes the delivery order. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * feat: count the occurrences a schedule skipped A schedule that overruns its interval, or waits for a worker, silently loses the occurrences in between: the scheduler keeps one queued occurrence and re-anchors on the clock, so nothing records that a run was due and never happened. Recovers the sequence from rows that already exist rather than writing per occurrence. `push_scheduled_job` anchors on `now_from_db` inside the transaction that inserts the job, and `v2_job.created_at` defaults to that same transaction timestamp, so `scheduled_for = find_next(created_at)` holds exactly and the whole occurrence history is derivable. The schedules list reports how many of the recent runs were followed by a lost occurrence, and a new occurrences endpoint carries the per-run wait and duration behind it. Detection is one `find_next` per gap, which stays bounded on a full page; counting walks the gap and runs only for a single schedule. The one write is `occurrence_baseline_at`, advanced at create, edit, re-enable and re-arm. Gaps older than it span a pause, a cron change, a re-enable or a reconciler re-arm, none of which mean runs were lost. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * feat: show the wait and run time behind a schedule's skipped occurrences The list badge says a schedule is losing runs; this says which of the two causes did it. A large wait means not enough workers, a long run means the job outgrew its interval, and the pair is what tells them apart. Sits under the existing upcoming-events panel, so due and overdue read together. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * feat: flag a schedule that is running late right now Reconstruction is retrospective: a gap only appears once the next occurrence has a row, which needs the current one to finish. A schedule wedged mid-run shows nothing until it moves, which is the case an operator most wants to see. An occurrence still in flight past the time its own successor was due will cost that successor, so `now > find_next(scheduled_for)` is the signal, needing no threshold and self-calibrating across a daily and a per-minute schedule. It applies only where occurrences serialize; an overlapping schedule starts its successor on time and would flag constantly while healthy. The queue is read in one aggregating pass keyed on (trigger, runnable_path) rather than a subquery per schedule, and an overlapping schedule holds more than one root row, hence the aggregate. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * feat: run the schedule overrun alert from the monitor pass Wires `schedule_overrun_alerts` in next to `jobs_waiting_alerts`, every 30 iterations (~5 min). Its Enterprise implementation lives in windmill-labs/windmill-ee-private#772; only the wiring and the OSS stub are here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * chore: refresh the sqlx offline cache Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * feat: record and alert when a schedule skips occurrences push_scheduled_job compares each chained occurrence with the slot after the previous one. A gap is written to schedule.skipped_occurrences off the push transaction, alerts once when a clean schedule starts skipping, and recovers on the next clean chain. The schedules list shows a badge. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * feat: alert only on a streak of skipping runs, keep a recent skip visible The skip state now describes the current streak and is written in the push transaction, so it commits or rolls back with the push. The alert fires once when 3 runs in a row skipped, and the list keeps a muted badge for 7 days after the latest skip. Editing or toggling a schedule resets it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * fix: name the missed-occurrence state after what it counts, alert only once committed Renames the columns to late_run_streak, missed_occurrences and last_missed_at, keeps the missed count after a streak ends so the muted badge can show it, and rewords both badges. The alert task now reads the streak FOR SHARE, which waits for the push transaction, so a push that rolls back and retries alerts once. A failed slot count leaves the streak untouched, and a schedule deleted mid-push no longer fails it. Adds an integration test for the streak and its reset. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * fix: recover the late run alert, store the missed slot, name it missed throughout The alert now recovers (and so acknowledges itself) when a streak that alerted ends on a run on time, under the schedule:{path} resource used by the other trigger alerts. last_missed_at records the last missed cron slot rather than when the late run chained, and the counting helpers say missed, since skipped already names occurrences queued and not run. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * fix: scope the late run alert to its workspace, acknowledge it on edit, toggle and delete Recovery acknowledges alerts by resource alone, so the resource now carries the workspace. Editing, toggling or deleting a schedule clears its streak and a disabled or deleted one never chains a run on time, so those handlers acknowledge its open alert after committing. Past the 1000-slot cap, last_missed_at falls back to the detection time. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 * refactor: raise the late run alert like the other critical alerts Drops the recovery, the workspace-scoped resource and the acknowledgement on edit, toggle and delete: the alert now fires once per streak with no resource and is acknowledged from the alerts feed, as the trigger and job failure alerts are. The FOR SHARE read stays, so a push that rolls back across the flow path's retries still alerts once. Notes in openapi that past 1000 misses in one late run the count is a floor and the time approximate. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01LJ9wpjWp2YgLUSqt1Ai5d6 --------- Co-authored-by: Claude Opus 5 --- ...d76a4e5702c0324a27ef3a5740ef5d98d8946.json | 31 ---- ...bed4c3d8b025947becd1c614757e7e29a5d8.json} | 12 +- ...bc3553985b757b48962e9976b5da5b812bd05.json | 49 ++++++ ...7128fa332a52e3b451aac307234d02a51de8f.json | 15 ++ ...b0b719f61fe0cb8a7a2c76be4b7422a7138a0.json | 23 +++ ...406bcbe888ddece7dec3330778591573031fb.json | 31 ++++ ...0a20126a207397cb9ed2dc18cd0b2baa57fa.json} | 12 +- ...c63922f0eebcba2b2cefc4290c4baee5c071.json} | 12 +- ...51151_schedule_missed_occurrences.down.sql | 3 + ...5151151_schedule_missed_occurrences.up.sql | 7 + backend/windmill-api-schedule/src/lib.rs | 24 ++- backend/windmill-api/openapi.yaml | 10 ++ backend/windmill-queue/src/schedule.rs | 141 +++++++++++++++++- backend/windmill-queue/tests/schedule_push.rs | 58 +++++++ backend/windmill-types/src/schedule.rs | 4 + .../components/schedules/SchedulesList.svelte | 34 ++++- 16 files changed, 416 insertions(+), 50 deletions(-) delete mode 100644 backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json rename backend/.sqlx/{query-2effe885aafaa0c5fc1f969b8c6ffbd38c53fb586251ba780b75a007e5a47ed6.json => query-1c51aab269efb558640ea502e63ebed4c3d8b025947becd1c614757e7e29a5d8.json} (95%) create mode 100644 backend/.sqlx/query-39793465641124401a732744777bc3553985b757b48962e9976b5da5b812bd05.json create mode 100644 backend/.sqlx/query-466b2c68074202146da10d4b0757128fa332a52e3b451aac307234d02a51de8f.json create mode 100644 backend/.sqlx/query-4a6fb702794ce250a8217034ae9b0b719f61fe0cb8a7a2c76be4b7422a7138a0.json create mode 100644 backend/.sqlx/query-640482b37fe2aa63a5082250170406bcbe888ddece7dec3330778591573031fb.json rename backend/.sqlx/{query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json => query-793810c58d9e872da52d478db61c0a20126a207397cb9ed2dc18cd0b2baa57fa.json} (79%) rename backend/.sqlx/{query-642ca096c1c151fc01445593c8d2ed879a9a7be6ff044a9ce1b581ba725665fe.json => query-ca54e732d521416cb2692c8c4406c63922f0eebcba2b2cefc4290c4baee5c071.json} (73%) create mode 100644 backend/migrations/20260925151151_schedule_missed_occurrences.down.sql create mode 100644 backend/migrations/20260925151151_schedule_missed_occurrences.up.sql diff --git a/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json deleted file mode 100644 index 0dbcf04c6e..0000000000 --- a/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4)\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "jobs", - "type_info": "JsonArray" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946" -} diff --git a/backend/.sqlx/query-2effe885aafaa0c5fc1f969b8c6ffbd38c53fb586251ba780b75a007e5a47ed6.json b/backend/.sqlx/query-1c51aab269efb558640ea502e63ebed4c3d8b025947becd1c614757e7e29a5d8.json similarity index 95% rename from backend/.sqlx/query-2effe885aafaa0c5fc1f969b8c6ffbd38c53fb586251ba780b75a007e5a47ed6.json rename to backend/.sqlx/query-1c51aab269efb558640ea502e63ebed4c3d8b025947becd1c614757e7e29a5d8.json index 6d948ee8ae..6c4d73699b 100644 --- a/backend/.sqlx/query-2effe885aafaa0c5fc1f969b8c6ffbd38c53fb586251ba780b75a007e5a47ed6.json +++ b/backend/.sqlx/query-1c51aab269efb558640ea502e63ebed4c3d8b025947becd1c614757e7e29a5d8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, args, enabled, email, permissioned_as,\n on_failure, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery, on_recovery_times, on_recovery_extra_args,\n on_success, on_success_extra_args,\n ws_error_handler_muted, retry, summary, no_flow_overlap,\n tag, paused_until, cron_version, description, dynamic_skip, labels\n ) VALUES (\n $1, $2, $3, $4, $5, $6,\n $7, $8, $9, $10, $11,\n $12, $13, $14, $15,\n $16, $17, $18,\n $19, $20,\n $21, $22, $23, $24,\n $25, $26, $27, $28, $29, $30\n )\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels\n ", + "query": "\n INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, args, enabled, email, permissioned_as,\n on_failure, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery, on_recovery_times, on_recovery_extra_args,\n on_success, on_success_extra_args,\n ws_error_handler_muted, retry, summary, no_flow_overlap,\n tag, paused_until, cron_version, description, dynamic_skip, labels\n ) VALUES (\n $1, $2, $3, $4, $5, $6,\n $7, $8, $9, $10, $11,\n $12, $13, $14, $15,\n $16, $17, $18,\n $19, $20,\n $21, $22, $23, $24,\n $25, $26, $27, $28, $29, $30\n )\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels,\n late_run_streak\n ", "describe": { "columns": [ { @@ -167,6 +167,11 @@ "ordinal": 32, "name": "labels", "type_info": "TextArray" + }, + { + "ordinal": 33, + "name": "late_run_streak", + "type_info": "Int4" } ], "parameters": { @@ -236,8 +241,9 @@ true, true, true, - true + true, + false ] }, - "hash": "2effe885aafaa0c5fc1f969b8c6ffbd38c53fb586251ba780b75a007e5a47ed6" + "hash": "1c51aab269efb558640ea502e63ebed4c3d8b025947becd1c614757e7e29a5d8" } diff --git a/backend/.sqlx/query-39793465641124401a732744777bc3553985b757b48962e9976b5da5b812bd05.json b/backend/.sqlx/query-39793465641124401a732744777bc3553985b757b48962e9976b5da5b812bd05.json new file mode 100644 index 0000000000..e1fe716eab --- /dev/null +++ b/backend/.sqlx/query-39793465641124401a732744777bc3553985b757b48962e9976b5da5b812bd05.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs, schedule.late_run_streak, schedule.missed_occurrences, schedule.last_missed_at FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4)\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + }, + { + "ordinal": 2, + "name": "late_run_streak", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "missed_occurrences", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "last_missed_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null, + false, + false, + true + ] + }, + "hash": "39793465641124401a732744777bc3553985b757b48962e9976b5da5b812bd05" +} diff --git a/backend/.sqlx/query-466b2c68074202146da10d4b0757128fa332a52e3b451aac307234d02a51de8f.json b/backend/.sqlx/query-466b2c68074202146da10d4b0757128fa332a52e3b451aac307234d02a51de8f.json new file mode 100644 index 0000000000..814659877f --- /dev/null +++ b/backend/.sqlx/query-466b2c68074202146da10d4b0757128fa332a52e3b451aac307234d02a51de8f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE schedule SET late_run_streak = 0\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "466b2c68074202146da10d4b0757128fa332a52e3b451aac307234d02a51de8f" +} diff --git a/backend/.sqlx/query-4a6fb702794ce250a8217034ae9b0b719f61fe0cb8a7a2c76be4b7422a7138a0.json b/backend/.sqlx/query-4a6fb702794ce250a8217034ae9b0b719f61fe0cb8a7a2c76be4b7422a7138a0.json new file mode 100644 index 0000000000..552f2f57de --- /dev/null +++ b/backend/.sqlx/query-4a6fb702794ce250a8217034ae9b0b719f61fe0cb8a7a2c76be4b7422a7138a0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT late_run_streak FROM schedule WHERE workspace_id = $1 AND path = $2 FOR SHARE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "late_run_streak", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4a6fb702794ce250a8217034ae9b0b719f61fe0cb8a7a2c76be4b7422a7138a0" +} diff --git a/backend/.sqlx/query-640482b37fe2aa63a5082250170406bcbe888ddece7dec3330778591573031fb.json b/backend/.sqlx/query-640482b37fe2aa63a5082250170406bcbe888ddece7dec3330778591573031fb.json new file mode 100644 index 0000000000..72699120db --- /dev/null +++ b/backend/.sqlx/query-640482b37fe2aa63a5082250170406bcbe888ddece7dec3330778591573031fb.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE schedule SET late_run_streak = late_run_streak + 1,\n missed_occurrences = CASE WHEN late_run_streak = 0 THEN $3\n ELSE missed_occurrences + $3 END,\n last_missed_at = $4\n WHERE workspace_id = $1 AND path = $2\n RETURNING late_run_streak, missed_occurrences", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "late_run_streak", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "missed_occurrences", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int4", + "Timestamptz" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "640482b37fe2aa63a5082250170406bcbe888ddece7dec3330778591573031fb" +} diff --git a/backend/.sqlx/query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json b/backend/.sqlx/query-793810c58d9e872da52d478db61c0a20126a207397cb9ed2dc18cd0b2baa57fa.json similarity index 79% rename from backend/.sqlx/query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json rename to backend/.sqlx/query-793810c58d9e872da52d478db61c0a20126a207397cb9ed2dc18cd0b2baa57fa.json index 5c0e45b0d4..b5afa669f5 100644 --- a/backend/.sqlx/query-4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56.json +++ b/backend/.sqlx/query-793810c58d9e872da52d478db61c0a20126a207397cb9ed2dc18cd0b2baa57fa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE schedule SET\n schedule = $1,\n timezone = $2,\n args = $3,\n on_failure = $4,\n on_failure_times = $5,\n on_failure_exact = $6,\n on_failure_extra_args = $7,\n on_recovery = $8,\n on_recovery_times = $9,\n on_recovery_extra_args = $10,\n on_success = $11,\n on_success_extra_args = $12,\n ws_error_handler_muted = $13,\n retry = $14,\n summary = $15,\n no_flow_overlap = $16,\n tag = $17,\n paused_until = $18,\n path = $19,\n workspace_id = $20,\n cron_version = COALESCE($21, cron_version),\n description = $22,\n dynamic_skip = $23,\n email = $24,\n edited_by = $25,\n edited_at = now(),\n permissioned_as = $26,\n labels = COALESCE($27, labels)\n WHERE path = $19 AND workspace_id = $20\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels\n ", + "query": "\n UPDATE schedule SET\n schedule = $1,\n timezone = $2,\n args = $3,\n on_failure = $4,\n on_failure_times = $5,\n on_failure_exact = $6,\n on_failure_extra_args = $7,\n on_recovery = $8,\n on_recovery_times = $9,\n on_recovery_extra_args = $10,\n on_success = $11,\n on_success_extra_args = $12,\n ws_error_handler_muted = $13,\n retry = $14,\n summary = $15,\n no_flow_overlap = $16,\n tag = $17,\n paused_until = $18,\n path = $19,\n workspace_id = $20,\n cron_version = COALESCE($21, cron_version),\n description = $22,\n dynamic_skip = $23,\n email = $24,\n edited_by = $25,\n edited_at = now(),\n permissioned_as = $26,\n labels = COALESCE($27, labels),\n late_run_streak = 0,\n missed_occurrences = 0,\n last_missed_at = NULL\n WHERE path = $19 AND workspace_id = $20\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels,\n late_run_streak\n ", "describe": { "columns": [ { @@ -167,6 +167,11 @@ "ordinal": 32, "name": "labels", "type_info": "TextArray" + }, + { + "ordinal": 33, + "name": "late_run_streak", + "type_info": "Int4" } ], "parameters": { @@ -233,8 +238,9 @@ true, true, true, - true + true, + false ] }, - "hash": "4d2c91937d87d385290b918af0238825cdd7784d0d611cd0069b15d11039ac56" + "hash": "793810c58d9e872da52d478db61c0a20126a207397cb9ed2dc18cd0b2baa57fa" } diff --git a/backend/.sqlx/query-642ca096c1c151fc01445593c8d2ed879a9a7be6ff044a9ce1b581ba725665fe.json b/backend/.sqlx/query-ca54e732d521416cb2692c8c4406c63922f0eebcba2b2cefc4290c4baee5c071.json similarity index 73% rename from backend/.sqlx/query-642ca096c1c151fc01445593c8d2ed879a9a7be6ff044a9ce1b581ba725665fe.json rename to backend/.sqlx/query-ca54e732d521416cb2692c8c4406c63922f0eebcba2b2cefc4290c4baee5c071.json index a205bcbbe6..0df21a408a 100644 --- a/backend/.sqlx/query-642ca096c1c151fc01445593c8d2ed879a9a7be6ff044a9ce1b581ba725665fe.json +++ b/backend/.sqlx/query-ca54e732d521416cb2692c8c4406c63922f0eebcba2b2cefc4290c4baee5c071.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE schedule SET\n enabled = $1,\n email = $2\n WHERE path = $3 AND workspace_id = $4\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels\n ", + "query": "\n UPDATE schedule SET\n enabled = $1,\n email = $2,\n late_run_streak = 0,\n missed_occurrences = 0,\n last_missed_at = NULL\n WHERE path = $3 AND workspace_id = $4\n RETURNING\n workspace_id,\n path,\n edited_by,\n edited_at,\n schedule,\n timezone,\n enabled,\n script_path,\n is_flow,\n args AS \"args: _\",\n extra_perms,\n email,\n permissioned_as,\n error,\n on_failure,\n on_failure_times,\n on_failure_exact,\n on_failure_extra_args AS \"on_failure_extra_args: _\",\n on_recovery,\n on_recovery_times,\n on_recovery_extra_args AS \"on_recovery_extra_args: _\",\n on_success,\n on_success_extra_args AS \"on_success_extra_args: _\",\n ws_error_handler_muted,\n retry,\n no_flow_overlap,\n summary,\n description,\n tag,\n paused_until,\n cron_version,\n dynamic_skip,\n labels,\n late_run_streak\n ", "describe": { "columns": [ { @@ -167,6 +167,11 @@ "ordinal": 32, "name": "labels", "type_info": "TextArray" + }, + { + "ordinal": 33, + "name": "late_run_streak", + "type_info": "Int4" } ], "parameters": { @@ -210,8 +215,9 @@ true, true, true, - true + true, + false ] }, - "hash": "642ca096c1c151fc01445593c8d2ed879a9a7be6ff044a9ce1b581ba725665fe" + "hash": "ca54e732d521416cb2692c8c4406c63922f0eebcba2b2cefc4290c4baee5c071" } diff --git a/backend/migrations/20260925151151_schedule_missed_occurrences.down.sql b/backend/migrations/20260925151151_schedule_missed_occurrences.down.sql new file mode 100644 index 0000000000..b89d6340ae --- /dev/null +++ b/backend/migrations/20260925151151_schedule_missed_occurrences.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE schedule DROP COLUMN last_missed_at; +ALTER TABLE schedule DROP COLUMN missed_occurrences; +ALTER TABLE schedule DROP COLUMN late_run_streak; diff --git a/backend/migrations/20260925151151_schedule_missed_occurrences.up.sql b/backend/migrations/20260925151151_schedule_missed_occurrences.up.sql new file mode 100644 index 0000000000..469eed3a93 --- /dev/null +++ b/backend/migrations/20260925151151_schedule_missed_occurrences.up.sql @@ -0,0 +1,7 @@ +-- A late run finished (or, for flows, started) after the next occurrence was due, so that +-- occurrence was missed. late_run_streak counts late runs in a row and is reset by a run on +-- time; missed_occurrences is what the latest streak missed and last_missed_at when, both +-- kept after the streak ends so a recent one stays visible. An edit or toggle clears all. +ALTER TABLE schedule ADD COLUMN late_run_streak INTEGER NOT NULL DEFAULT 0; +ALTER TABLE schedule ADD COLUMN missed_occurrences INTEGER NOT NULL DEFAULT 0; +ALTER TABLE schedule ADD COLUMN last_missed_at TIMESTAMPTZ; diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 78cd6eb378..7a523c89fa 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -409,7 +409,8 @@ async fn create_schedule( paused_until, cron_version, dynamic_skip, - labels + labels, + late_run_streak "#, w_id, ns.path, @@ -621,7 +622,10 @@ async fn edit_schedule( edited_by = $25, edited_at = now(), permissioned_as = $26, - labels = COALESCE($27, labels) + labels = COALESCE($27, labels), + late_run_streak = 0, + missed_occurrences = 0, + last_missed_at = NULL WHERE path = $19 AND workspace_id = $20 RETURNING workspace_id, @@ -656,7 +660,8 @@ async fn edit_schedule( paused_until, cron_version, dynamic_skip, - labels + labels, + late_run_streak "#, es.schedule, es.timezone, @@ -1004,6 +1009,9 @@ async fn list_schedule( pub struct ScheduleWJobs { pub path: String, pub jobs: Option>, + pub late_run_streak: i32, + pub missed_occurrences: i32, + pub last_missed_at: Option>, } async fn list_schedule_with_jobs( @@ -1020,7 +1028,7 @@ async fn list_schedule_with_jobs( // - use of the `ix_v2_job_root_by_path` index; hence the `parent_job IS NULL` clause. // - both `workspace_id = $1` checks are required to hit both indexes. "SELECT - schedule.path, t.jobs FROM schedule, + schedule.path, t.jobs, schedule.late_run_streak, schedule.missed_occurrences, schedule.last_missed_at FROM schedule, LATERAL(SELECT ARRAY( SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms) FROM v2_job_completed c JOIN v2_job j USING (id) @@ -1157,7 +1165,10 @@ pub async fn set_enabled( r#" UPDATE schedule SET enabled = $1, - email = $2 + email = $2, + late_run_streak = 0, + missed_occurrences = 0, + last_missed_at = NULL WHERE path = $3 AND workspace_id = $4 RETURNING workspace_id, @@ -1192,7 +1203,8 @@ pub async fn set_enabled( paused_until, cron_version, dynamic_skip, - labels + labels, + late_run_streak "#, payload.enabled, authed.email, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 00fb3e07de..4a1ce62bad 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -32107,6 +32107,16 @@ components: - id - success - duration_ms + late_run_streak: + type: integer + description: runs in a row, up to the latest, that started or finished after the next occurrence was due, so that occurrence was missed + missed_occurrences: + type: integer + description: occurrences missed by the latest streak of late runs, kept after it ends; a lower bound once a single late run misses 1000 + last_missed_at: + type: string + format: date-time + description: latest missed occurrence, kept after the schedule runs on time again; past 1000 misses in one late run, when the miss was detected, at most one period later ErrorHandler: type: string diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 40c1292b74..97b175fa9e 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -36,7 +36,7 @@ use windmill_common::DB; use windmill_common::{ error::{self, Result}, schedule::Schedule, - utils::{now_from_db, ScheduleType, StripPath}, + utils::{now_from_db, report_critical_error, ScheduleType, StripPath}, }; /// Helper to fetch metadata for a schedule's script or flow @@ -203,6 +203,62 @@ pub async fn push_scheduled_job<'c>( return Ok(tx); } + // Only a chained push (`now_cutoff` is the previous occurrence) can miss one: create, + // edit, enable and re-arm start fresh, and a pause, even one already over, is deliberate. + let missed = now_cutoff + .filter(|_| schedule.paused_until.is_none()) + .and_then(|prev| { + // A failed count leaves the streak as it was rather than reading as a run on time. + count_missed_occurrences(&sched, &tz, prev, next) + .inspect_err(|e| { + tracing::warn!( + "failed to count the occurrences schedule {} missed: {e}", + &schedule.path + ) + }) + .ok() + }); + if let Some((missed, last_missed)) = missed { + // Past the cap the walk stops short of the last miss; the time it was detected is at + // most one period after it. + let last_missed = last_missed.map(|l| if missed == MAX_COUNTED_MISSES { now } else { l }); + if let Some(last_missed) = last_missed { + let streak = sqlx::query!( + "UPDATE schedule SET late_run_streak = late_run_streak + 1, + missed_occurrences = CASE WHEN late_run_streak = 0 THEN $3 + ELSE missed_occurrences + $3 END, + last_missed_at = $4 + WHERE workspace_id = $1 AND path = $2 + RETURNING late_run_streak, missed_occurrences", + &schedule.workspace_id, + &schedule.path, + missed as i32, + last_missed, + ) + .fetch_optional(&mut *tx) + .warn_after_seconds_with_sql(1, "update_schedule_late_run_streak".to_string()) + .await?; + if let Some(streak) = streak.filter(|s| s.late_run_streak == LATE_RUNS_BEFORE_ALERT) { + tokio::spawn(alert_late_run_streak( + db.clone(), + schedule.workspace_id.clone(), + schedule.path.clone(), + streak.missed_occurrences, + )); + } + } else if schedule.late_run_streak > 0 { + sqlx::query!( + "UPDATE schedule SET late_run_streak = 0 + WHERE workspace_id = $1 AND path = $2", + &schedule.workspace_id, + &schedule.path, + ) + .execute(&mut *tx) + .warn_after_seconds_with_sql(1, "reset_schedule_late_run_streak".to_string()) + .await?; + } + } + let mut args: HashMap> = HashMap::new(); if let Some(args_v) = &schedule.args { @@ -622,6 +678,68 @@ pub async fn push_scheduled_job<'c>( Ok(tx) // TODO: Bubble up pushed UUID from here } +const MAX_COUNTED_MISSES: u32 = 1000; + +/// Due slots of the cron strictly between the previous occurrence and the next one, and +/// the last of them. A chain on time costs one `find_next`: its first slot is `next` itself. +fn count_missed_occurrences( + sched: &ScheduleType, + tz: &chrono_tz::Tz, + prev: DateTime, + next: DateTime, +) -> Result<(u32, Option>)> { + let mut count = 0; + let mut last = None; + let mut slot = prev.with_timezone(tz); + while count < MAX_COUNTED_MISSES { + match sched.find_next(&slot) { + Ok(s) if s.with_timezone(&Utc) < next => { + count += 1; + last = Some(s.with_timezone(&Utc)); + slot = s; + } + Ok(_) => break, + Err(e) => return Err(e), + } + } + Ok((count, last)) +} + +/// A single late run is a blip (a slow run, a worker restart, an edit mid-run); only a +/// streak alerts, once, when it reaches this length. The schedules list shows every one. +const LATE_RUNS_BEFORE_ALERT: i32 = 3; + +/// Spawned: it reaches the instance alert channels, which must not hold up the push. +async fn alert_late_run_streak(db: DB, w_id: String, path: String, missed: i32) { + // The push transaction holds this row until it ends, so FOR SHARE waits for it: a push + // that rolled back leaves the streak short of the threshold, and only its committed + // retry alerts. + let committed = sqlx::query_scalar!( + "SELECT late_run_streak FROM schedule WHERE workspace_id = $1 AND path = $2 FOR SHARE", + &w_id, + &path, + ) + .fetch_optional(&db) + .await; + match committed { + Ok(Some(streak)) if streak >= LATE_RUNS_BEFORE_ALERT => {} + Ok(_) => return, + Err(e) => { + tracing::warn!("failed to confirm the late run streak of schedule {w_id}/{path}: {e}"); + return; + } + } + report_critical_error( + format!( + "Schedule {path} missed {missed} occurrences: its last {LATE_RUNS_BEFORE_ALERT} runs in a row started or finished too late" + ), + db, + Some(&w_id), + None, + ) + .await; +} + /// Enabled schedules with no occurrence in the queue, as `(workspace_id, path)`. /// /// Every path that completes a scheduled job pushes the next occurrence in the @@ -743,7 +861,7 @@ pub async fn get_schedule_opt<'c>( path: &str, ) -> Result> { let schedule_opt = sqlx::query_as::<_, Schedule>( - "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule WHERE path = $1 AND workspace_id = $2", + "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels, late_run_streak FROM schedule WHERE path = $1 AND workspace_id = $2", ) .bind(path) .bind(w_id) @@ -802,3 +920,22 @@ pub async fn clear_schedule<'c>( windmill_common::jobs::delete_jobs(&mut **tx, &deleted_ids).await?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_the_slots_missed_between_two_occurrences() { + let every_30s = ScheduleType::from_str("*/30 * * * * *", None, false).unwrap(); + let at = |s: &str| DateTime::parse_from_rfc3339(s).unwrap().with_timezone(&Utc); + let tz = chrono_tz::UTC; + let prev = at("2026-09-02T08:20:00Z"); + let count = |next| count_missed_occurrences(&every_30s, &tz, prev, at(next)).unwrap(); + assert_eq!(count("2026-09-02T08:20:30Z"), (0, None)); + assert_eq!( + count("2026-09-02T08:21:30Z"), + (2, Some(at("2026-09-02T08:21:00Z"))) + ); + } +} diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index f3ae34251e..c68c59bb16 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -49,6 +49,7 @@ mod schedule_push { cron_version: None, dynamic_skip: None, labels: None, + late_run_streak: 0, }; overrides(&mut s); s @@ -511,6 +512,63 @@ mod schedule_push { Ok(()) } + // ----------------------------------------------------------------------- + // push_scheduled_job: late run streak + // ----------------------------------------------------------------------- + + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_push_tracks_late_run_streak(db: Pool) -> anyhow::Result<()> { + let mut schedule = make_schedule(|s| s.schedule = "0 0 0 * * *".to_string()); + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, schedule, script_path, permissioned_as) + VALUES ($1, $2, 'test-user', $3, $4, 'u/test-user')", + ) + .bind(&schedule.workspace_id) + .bind(&schedule.path) + .bind(&schedule.schedule) + .bind(&schedule.script_path) + .execute(&db) + .await?; + let authed = make_authed(); + let streak = || { + sqlx::query_as::<_, (i32, i32, Option>)>( + "SELECT late_run_streak, missed_occurrences, last_missed_at FROM schedule WHERE path = $1", + ) + .bind(&schedule.path) + .fetch_one(&db) + }; + let push = |schedule: Schedule, prev: chrono::DateTime| { + let db = db.clone(); + let authed = authed.clone(); + async move { + sqlx::query("DELETE FROM v2_job_queue").execute(&db).await?; + let tx = db.begin().await?; + push_scheduled_job(&db, tx, &schedule, Some(&authed), Some(prev)) + .await? + .commit() + .await?; + anyhow::Ok(()) + } + }; + + let midnight = Utc::now() + .date_naive() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(); + + // The daily run due 3 days ago finished just now: the 3 midnights since were missed. + push(schedule.clone(), midnight - chrono::Duration::days(3)).await?; + let (runs, missed, at) = streak().await?; + assert_eq!((runs, missed, at), (1, 3, Some(midnight))); + + // Today's run on time ends the streak but keeps what it missed, for the badge. + schedule.late_run_streak = runs; + push(schedule.clone(), midnight).await?; + assert_eq!(streak().await?, (0, 3, at)); + Ok(()) + } + // ----------------------------------------------------------------------- // try_schedule_next_job: disabled schedule does not push // ----------------------------------------------------------------------- diff --git a/backend/windmill-types/src/schedule.rs b/backend/windmill-types/src/schedule.rs index 75a48f8798..ce3add9605 100644 --- a/backend/windmill-types/src/schedule.rs +++ b/backend/windmill-types/src/schedule.rs @@ -57,6 +57,10 @@ pub struct Schedule { pub dynamic_skip: Option, #[serde(skip_serializing_if = "Option::is_none")] pub labels: Option>, + // Runtime state, not configuration: kept out of exports and git sync. + #[sqlx(default)] + #[serde(skip)] + pub late_run_streak: i32, } impl Schedule { diff --git a/frontend/src/lib/components/schedules/SchedulesList.svelte b/frontend/src/lib/components/schedules/SchedulesList.svelte index 2214a0d6a9..7935cd6725 100644 --- a/frontend/src/lib/components/schedules/SchedulesList.svelte +++ b/frontend/src/lib/components/schedules/SchedulesList.svelte @@ -6,7 +6,7 @@ type WorkspaceDeployUISettings, WorkspaceService } from '$lib/gen' - import { canWrite, displayDate, getLocalSetting, storeLocalSetting } from '$lib/utils' + import { canWrite, displayDate, getLocalSetting, pluralize, storeLocalSetting } from '$lib/utils' import { useScheduleLock } from '$lib/operatorWriteRights' import { withForkConflictRetry } from '$lib/utils/forkConflict' import { base } from '$app/paths' @@ -34,7 +34,8 @@ Plus, SearchX, Shield, - Trash + Trash, + TriangleAlert } from 'lucide-svelte' import { goto } from '$lib/navigation' import { sendUserToast } from '$lib/toast' @@ -77,6 +78,8 @@ type ScheduleW = ScheduleWJobs & { canWrite: boolean } + const MISSED_VISIBLE_MS = 7 * 24 * 60 * 60 * 1000 + let schedules: ScheduleW[] = $state([]) let shareModal: ShareModal | undefined = $state() let loading = $state(true) @@ -170,6 +173,9 @@ for (let schedule of schedules) { if (schedulesWithJobsByPath[schedule.path]) { schedule.jobs = schedulesWithJobsByPath[schedule.path].jobs + schedule.late_run_streak = schedulesWithJobsByPath[schedule.path].late_run_streak + schedule.missed_occurrences = schedulesWithJobsByPath[schedule.path].missed_occurrences + schedule.last_missed_at = schedulesWithJobsByPath[schedule.path].last_missed_at } } loadingSchedulesWithJobStats = false @@ -392,6 +398,9 @@ extra_perms, canWrite, jobs, + late_run_streak, + missed_occurrences, + last_missed_at, paused_until, labels, inherited_labels, @@ -484,6 +493,27 @@
    {/snippet} + {:else if late_run_streak} + + + {#snippet text()} +
    + Missed {pluralize(missed_occurrences ?? 0, 'occurrence')}: the last + {late_run_streak === 1 ? 'run' : `${late_run_streak} runs in a row`} started or finished + too late. +
    + {/snippet} +     + {:else if last_missed_at && Date.now() - new Date(last_missed_at).getTime() < MISSED_VISIBLE_MS} + + + {#snippet text()} +
    + Missed {pluralize(missed_occurrences ?? 0, 'occurrence')}, the last on + {displayDate(last_missed_at)}. On schedule since. +
    + {/snippet} +     {/if}
    From 797147ea7a4a911d3c82e9ac9cdaf066ad2795d6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 29 Sep 2026 14:27:23 +0200 Subject: [PATCH 68/91] fix: report a worker's last job when it ran under one poll interval (#11399) * fix: report a worker's last job when it ran under one poll interval Co-Authored-By: Claude Opus 5.5 (1M context) * fix: share the unreported job slot with the interactive worker shell Co-Authored-By: Claude Opus 5.5 (1M context) * test: pin that a main-loop ping without a job keeps the last one Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- ...e424819133ce38dab7300a5fca4c6f54536d.json} | 6 ++- backend/tests/worker_ping_ip.rs | 50 ++++++++++++++++++- backend/windmill-common/src/worker.rs | 10 +++- backend/windmill-worker/src/worker.rs | 12 +++++ backend/windmill-worker/src/worker_utils.rs | 8 ++- 5 files changed, 80 insertions(+), 6 deletions(-) rename backend/.sqlx/{query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json => query-d267e03115ee74ab7ca7edd8a010e424819133ce38dab7300a5fca4c6f54536d.json} (70%) diff --git a/backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json b/backend/.sqlx/query-d267e03115ee74ab7ca7edd8a010e424819133ce38dab7300a5fca4c6f54536d.json similarity index 70% rename from backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json rename to backend/.sqlx/query-d267e03115ee74ab7ca7edd8a010e424819133ce38dab7300a5fca4c6f54536d.json index ceb05001e4..f3958e713d 100644 --- a/backend/.sqlx/query-0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626.json +++ b/backend/.sqlx/query-d267e03115ee74ab7ca7edd8a010e424819133ce38dab7300a5fca4c6f54536d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip) WHERE worker = $6", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip),\n current_job_id = COALESCE($14, current_job_id), current_job_workspace_id = COALESCE($15, current_job_workspace_id) WHERE worker = $6", "describe": { "columns": [], "parameters": { @@ -17,10 +17,12 @@ "Float4", "Float4", "Bool", + "Varchar", + "Uuid", "Varchar" ] }, "nullable": [] }, - "hash": "0c18351237816fe0c56e23801fcb8e70dbffcf08ed121e55c871f727c4ddf626" + "hash": "d267e03115ee74ab7ca7edd8a010e424819133ce38dab7300a5fca4c6f54536d" } diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs index 8f98398b40..772f4193c3 100644 --- a/backend/tests/worker_ping_ip.rs +++ b/backend/tests/worker_ping_ip.rs @@ -1,5 +1,9 @@ use sqlx::{Pool, Postgres}; -use windmill_common::{external_ip::UNKNOWN_IP, worker::insert_ping_query}; +use uuid::Uuid; +use windmill_common::{ + external_ip::UNKNOWN_IP, + worker::{insert_ping_query, update_worker_ping_main_loop_query}, +}; async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { insert_ping_query( @@ -42,3 +46,47 @@ async fn unresolved_ip_keeps_the_reclaimed_rows_address(db: Pool) -> a assert_eq!(ip, UNKNOWN_IP); Ok(()) } + +async fn main_loop_ping( + db: &Pool, + worker: &str, + last_job: Option<(Uuid, &str)>, +) -> anyhow::Result<()> { + update_worker_ping_main_loop_query( + worker, + &[], + None, + None, + Some(1), + None, + None, + None, + None, + None, + None, + false, + None, + last_job, + db, + ) + .await +} + +/// The main-loop ping sends a job only once, so most pings carry none. Those must keep the last +/// job the row holds, whether this ping or the job poller wrote it. +#[sqlx::test] +async fn main_loop_ping_without_a_job_keeps_the_last_one(db: Pool) -> anyhow::Result<()> { + insert_ping(&db, "wk-last-job", None).await?; + let job_id = Uuid::new_v4(); + main_loop_ping(&db, "wk-last-job", Some((job_id, "admins"))).await?; + main_loop_ping(&db, "wk-last-job", None).await?; + let (current_job_id, workspace_id): (Option, Option) = sqlx::query_as( + "SELECT current_job_id, current_job_workspace_id FROM worker_ping WHERE worker = $1", + ) + .bind("wk-last-job") + .fetch_one(&db) + .await?; + assert_eq!(current_job_id, Some(job_id)); + assert_eq!(workspace_id.as_deref(), Some("admins")); + Ok(()) +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ad0a5068e8..d480a89851 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1939,6 +1939,9 @@ pub async fn update_ping_http( insert_ping.occupancy_rate_30m, insert_ping.native_mode.unwrap_or(false), insert_ping.ip.as_deref(), + insert_ping + .last_job_executed + .zip(insert_ping.last_job_workspace_id.as_deref()), db, ) .await? @@ -2220,12 +2223,15 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_30m: Option, native_mode: bool, ip: Option<&str>, + last_job: Option<(Uuid, &str)>, db: &DB, ) -> anyhow::Result<()> { + let (last_job_id, last_job_workspace_id) = last_job.unzip(); timeout(Duration::from_secs(10), sqlx::query!( "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), - memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip) WHERE worker = $6", + memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11, native_mode = $12, ip = COALESCE($13, ip), + current_job_id = COALESCE($14, current_job_id), current_job_workspace_id = COALESCE($15, current_job_workspace_id) WHERE worker = $6", jobs_executed, tags, occupancy_rate, @@ -2239,6 +2245,8 @@ pub async fn update_worker_ping_main_loop_query( occupancy_rate_30m, native_mode, ip, + last_job_id, + last_job_workspace_id, ) .execute(db)) .await??; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a2ddac713e..1b4cfdd771 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2321,6 +2321,7 @@ fn start_interactive_worker_shell( job_completed_tx: JobCompletedSender, base_internal_url: String, worker_dir: String, + unreported_job: Arc>>, ) -> JoinHandle<()> { tokio::spawn(async move { let started_at = Instant::now(); @@ -2407,6 +2408,8 @@ fn start_interactive_worker_shell( flow_runners, } = extract_job_and_perms(job, &conn).await; + *unreported_job.lock().unwrap() = Some((job.id, job.workspace_id.clone())); + let authed_client = AuthedClient::new( base_internal_url.to_owned(), job.workspace_id.clone(), @@ -2899,6 +2902,11 @@ pub async fn run_worker( // Only jobs run by this process count towards EXIT_AFTER_N_JOBS: the point is the age of // the environment, not the lifetime total. let mut jobs_executed_in_env: u64 = 0; + // The job poller only records a job in `worker_ping` once it has run for a poll interval, so + // the main-loop ping reports the last pulled job for shorter ones. The interactive shell runs + // jobs under this worker's name too and must set the same slot, or the ping would overwrite a + // shell job its poller recorded with an older main-loop job. Sent once, then left to the poller. + let unreported_job: Arc>> = Default::default(); let is_dedicated_worker: bool = { let config = WORKER_CONFIG.load(); @@ -3016,6 +3024,7 @@ pub async fn run_worker( job_completed_tx.clone(), base_internal_url.to_owned(), worker_dir.clone(), + unreported_job.clone(), ); Some(it_shell) @@ -3219,6 +3228,7 @@ pub async fn run_worker( let read_cgroups = *REFRESH_CGROUP_READINGS && last_reading.elapsed().as_secs() > NUM_SECS_READINGS; + let last_job = unreported_job.lock().unwrap().take(); update_worker_ping_full( &conn, read_cgroups, @@ -3228,6 +3238,7 @@ pub async fn run_worker( &mut occupancy_metrics, &killpill_tx, ip, + last_job.as_ref().map(|(id, w_id)| (*id, w_id.as_str())), ) .await; @@ -3562,6 +3573,7 @@ pub async fn run_worker( last_executed_job = None; jobs_executed += 1; + *unreported_job.lock().unwrap() = Some((job.id, job.workspace_id.clone())); let mut dirties_env = dirties_worker_env( job.kind, &job.tag, diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 67e9831481..3a313e9360 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -28,6 +28,7 @@ pub(crate) async fn update_worker_ping_full( occupancy_metrics: &mut OccupancyMetrics, killpill_tx: &KillpillSender, ip: Option<&str>, + last_job: Option<(Uuid, &str)>, ) { let wc = WORKER_CONFIG.load(); let tags = wc.worker_tags.clone(); @@ -67,6 +68,7 @@ pub(crate) async fn update_worker_ping_full( occupancy_rate_30m, native_mode, ip, + last_job, ) }) .retry( @@ -114,6 +116,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_30m: Option, native_mode: bool, ip: Option<&str>, + last_job: Option<(Uuid, &str)>, ) -> anyhow::Result<()> { match conn { Connection::Sql(db) => { @@ -131,6 +134,7 @@ async fn update_worker_ping_full_inner( occupancy_rate_30m, native_mode, ip, + last_job, db, ) .await?; @@ -141,8 +145,8 @@ async fn update_worker_ping_full_inner( UPDATE_PING_URL, None, &Ping { - last_job_executed: None, - last_job_workspace_id: None, + last_job_executed: last_job.map(|(id, _)| id), + last_job_workspace_id: last_job.map(|(_, w_id)| w_id.to_string()), worker_instance: None, ip: ip.map(str::to_string), tags: Some(tags.to_vec()), From 8741843d2e884783eed1c00bbc2293e526f3e80d Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:45:47 +0200 Subject: [PATCH 69/91] feat: external instance cluster for data tables and Ducklake catalogs (#11197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix pg_dump stuck on version 17 on nix * fix(datatables): refuse a malformed role annotation instead of ignoring it `-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed the annotation parser's exact-match rule, so the query fell through to the data table's default role and ran, silently, under a login the author did not choose. Naming a role exists precisely to not do that. A leading comment whose first word is `role` is now an annotation attempt: the keyword matches case-insensitively, one trailing `;` is tolerated, and anything else is an error naming the line. Only callers that already know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its comments. Also bumps the dev shell's postgres client to 18 — it trailed the server the dev database runs, which takes out every data table export, clone and fork-with-data. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse a malformed role query string instead of ignoring it `?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference parser's exact-match rule, so the connection resolved to the data table's default role and ran under a login the caller never asked for — the URI half of the same trap as a malformed `-- role` annotation. The key now matches case-insensitively, and anything else in the query string is an error naming it; `role` is the only parameter a reference takes. Callers that only need the entry keep a lenient `datatable_ref_name`, since they never act on the role. The DuckDB `ATTACH` parser propagates it rather than attaching under the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): carry the role annotation into the row_to_json retry The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment block never reached the second attempt — and with it the `-- role ` line that decides which login the query runs as. The retry connected as the data table's default role instead, so a query the first attempt was denied could succeed on the second, reported as "recovered with the row_to_json fix". Carry the leading comment block over. The retry itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore(datatables): don't mount the roles UI until the ACL editor lands Enforcement ships first. The permissions drawer is what turns roles on, and the catalog section is what creates them — both are only useful once there is a way to grant a role the privileges it needs, which arrives with the ACL editor. Left mounted they would offer a feature whose other half does not exist. The two components are complete and reviewed; only their call sites here are commented out, with a note pointing the follow-up PRs at them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): honour `-- role: x`, and fix the DuckDB attach test Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse to roll back the catalog while roles exist The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * refactor(datatables): settle clonability in one place, before anything is created A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): let a retried clone reclaim its own leftover database A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) * Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bounce the streams reading a data table when it is deleted Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) * refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. * fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): data table roles in the DB manager and raw apps Co-Authored-By: Claude Opus 5 (1M context) * fix: never add a role to the reference of a data table whose name contains '?' Co-Authored-By: Claude Opus 5 (1M context) * fix: read the roles of a data table whose name contains '?' The generated client leaves a '?' in a path param unencoded, so the lookup 404'd and the raw-app picker blocked Start on such a data table. Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): refuse a reference naming both a legacy data table and a role When a workspace stores both `sales` and a legacy `sales?role=analytics`, the reference resolved to the legacy entry without a role, so browsing `sales` as `analytics` reached another data table. Co-Authored-By: Claude Opus 5 (1M context) * fix: declare the default role in migrations written for a data table whose name contains '?' Such a data table connects as its default role without naming it, so the migrations the manager wrote for it declared no role and ran as admin. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * feat(datatables): set up an external instance cluster for data tables Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): send external cluster passwords as SCRAM verifiers Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): scope external cluster credential readers to the crate Co-Authored-By: Claude Opus 5 (1M context) * [ee] feat(datatables): external_instance data tables on the external cluster Co-Authored-By: Claude Opus 5 (1M context) * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): compare the external cluster settings under a row lock before storing setup Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): only drop external databases Windmill marked, and check use under the lock Co-Authored-By: Claude Opus 5 (1M context) * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * feat(datatables): put a data table's connection under Postgres roles A data table backed by the instance database resolved to exactly one Postgres connection, `custom_instance_user`, for everyone who could reach it at all. There was no way to say this job reads, that one writes, this one never sees the salaries table. A data table role is now a real Postgres login on the cluster, defined once for the instance by a superadmin and named exactly as they named it. A script that declares `-- role analytics` connects as `analytics`, and Postgres decides what it may touch — grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`. A data table with no `permissions` block behaves exactly as before. Everything that opens a connection on someone's behalf goes through one chokepoint, `get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when there is none. The role logs in as itself — never `SET ROLE`, which a script could `RESET ROLE` its way out of. A fork's data table entry becomes a pointer at the workspace that governs it rather than a copy of it. The settings clone used to hand a fork a byte-identical entry naming the parent's database, which a fork admin could edit to grant themselves `admin` there; a pointer has nothing local to edit, and its tenants are evaluated as a member of the governing workspace, by email. `permissions` is stripped from the workspace export and ignored on import: tenants name principals of one workspace, and a settings push is not where an access decision should be made. Operations that see the whole database whatever the roles grant stay with the governing workspace's admins: editing the roles, a migration that declares none, and opening a replication stream for a Postgres trigger or capture. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): gate the paths that reach a whole database as admin Auditing what still resolved through the unchecked resolver turned up three that act for a caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema export, the full-schema read, database creation, import and the forked-database drop), the connection test, and the schema snapshot a fork clone takes of its parent. On a data table under roles each let any workspace member — or a fork admin who is nobody in the governing workspace — read or copy the whole database whatever its roles grant. All three now require admin reach on the governing workspace. A dump taken under a restricted role would be a silently truncated copy rather than an error, so refusing is the only right answer for the copy paths. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin putting a *resource-backed* data table under roles, at which point the executor dialled the host that resource names — one the admin chose — with the role's real cluster password, and `CONNECT` is granted to every registered instance database. Both ends now refuse: the permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials on a non-instance entry rather than trusting the record it read. Two more places reached the governing database without answering to it. The initial-migration generator returned a `pg_dump` of the whole schema to any member. And the migration rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after which the parent re-runs every migration from zero. The remote half is now skipped when the entry resolves into another workspace, which is also just correct: a fork renaming what it calls a data table changes nothing about the data table. Also: revoking a tenant now bounces the replication streams of every workspace holding an entry that resolves here, not only the governing one, so a fork's trigger stops rather than living on inside its open connection; the instance role catalog and the governing workspace's tenant lists are no longer returned to someone who cannot edit them; and the tenant rename dedup collapses non-adjacent duplicates, per role rather than once any role changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): fail loudly where a role or a pointer can be left half-recorded Three ways the feature could end up in a state nobody could see or undo. Creating a role writes the cluster first and the catalog second, but the catalog write was an `UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a live login with a password nobody recorded: invisible to the catalog, un-recreatable because the name is taken, and un-deletable because there is no entry to delete. It now errors, so the operation is retryable once the row is restored. Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy would hand each fork the database outright — so the delete now names the data tables it stranded, and resolving one says which workspace is missing rather than reporting a data table this workspace never had. `InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now hand-written so `{:?}` on the catalog cannot put a live credential in a log line. Adds the two branches the reviews found unpinned: a caller who is not a member of the governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that predates this and sends no job id. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): unbreak two operator messages and two comments that described other code The two strings this branch added for states an operator hits once — the catalog write that matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence. `list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop it, and the comment now says why that is the right place to stay quiet. The non-superadmin check in `edit_datatable_config` was introduced as also covering references, which it does not and need not: `reference` is overwritten from the stored entry for every caller before the check runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): serialize role catalog mutations, and state each helper's authorization contract The catalog is one JSON document, so create, rename, enable and delete are all read-modify-write. Two concurrent creates read the same snapshot, both succeed in the cluster, and the second write drops the first — leaving a live Postgres login with a password nobody recorded, which is the exact state the delete path exists to prevent. Every mutation now runs in one transaction holding an advisory lock across the read, the cluster DDL and the write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL helpers take that transaction rather than the pool, which is what makes the lock cover them. Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely multi-statement SQL, and its future is not `Send`, which an axum handler holding the transaction requires. Each of these is one statement anyway. The new cross-crate surface now says what callers must do. `read_role_catalog` returns plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and `converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's raw config. All of them are superadmin-gated by their current handlers, but nothing said so at the definition, which is where the next caller looks. Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip that did not land; the rename affordance is the design-system `Button`, not a raw one; and `resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing resolves as the data table's default until the database manager grows a picker. Why role passwords stay a plain `String` while the instance user's password beside them is a `StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an operator supplies it and may want it from their own backend, while these are minted here and never entered by anyone, so there is nothing for a ref to point at. Encrypting generated secrets at rest is a separate change that would take the replication password with it. Now said at the field. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): give the role catalog its own row, out of reach of the config machinery Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways. The catalog serializes a generated Postgres password per role, and that row is the operator-facing instance config, so the passwords reached `get_instance_config` and its YAML editor — a live cluster credential in a response body, a UI field and any log of either. Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of that key writes the row back without it and the catalog is gone, while the cluster keeps every login it described. `custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret, written only by the server, never operator-authored, hidden so the config machinery cannot read, rewrite or drop it. The catalog is the same thing, so it now has the same shape: `datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist. No redaction to keep in step with three code paths, and no way for a neighbouring write to take it out. Two races on the same shared documents. `edit_datatable_config` read the stored data tables outside its transaction and then wrote the whole `datatable` document, so a permissions save committing in between was silently rolled back; it now reads under `FOR UPDATE`. And `set_datatable_permissions` validated role ids against the catalog before opening its transaction, so a deletion in between let it write a deleted role back — including as the default, which every later job then fails on; it now holds the catalog lock and the settings row across validation and write. Completes the authorization contracts the previous commit claimed but did not finish: `read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole job is to answer for a workspace the caller may not belong to, and `converge_connect_grants_with`, which had not inherited its wrapper's. Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but `extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so code written from that page passed a keyword-only argument positionally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all three handlers left it green — it pinned Postgres, not the code it was written for. It now drives `create_datatable_role` twice concurrently and asserts the catalog kept both names. Checked the way the last one should have been: removing the lock from the handler makes it fail with "wmtest_a_… is a live cluster login the catalog forgot". The contracts added last commit were stricter than this PR's own callers, which is worse than none — the next reader sees a rule already broken and learns to ignore it. `read_role_catalog` said superadmin-only while two of its four callers are open to any workspace member, and `converge_connect_grants` said superadmin while `set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the rule that actually holds is about the credential never reaching a response, log, audit record or export, not about who may call. They now say that. `read_datatable_entry` gets the same treatment rather than the one the earlier message claimed for it: it is the primitive every resolution goes through, so it is deliberately open, and what must not escape is `permissions` — it names the governing workspace's users, groups and folders. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): close the last ways a role or a pointer can be left pointing at nothing The raw settings readers hand back whatever is in the row, so moving the catalog into its own `global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles` and the settings listing returning every live password. Both now filter that one key. The neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it predates this and widening the fix to it is a decision about an operator workflow, not a consequence of this change. Three ways a save could leave something resolving to nothing: A permissioned data table could be moved to a PostgreSQL resource. The block was carried across as a server-owned field, the runtime refuses roles on a resource-backed table, so the save succeeded and every job afterwards failed. Refused instead — turning roles off first is one step, and it keeps discarding an access decision something somebody chose. Renaming a governing data table left every fork pointing at the old name: the data table disappears from their pickers and their jobs stop, with nothing in the renaming workspace to suggest why. The rename now follows into the pointers in the same transaction. Deleting one cannot be followed the same way, so it is reported instead — the response names what it stranded, the way deleting a workspace does, and the fork's own error already says which workspace is gone. Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions handler reaches it as a workspace admin (the same class fixed last commit, one instance missed); the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog` was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its roles only on success — a failing run is exactly the one that creates them without recording them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * refactor(datatables): put the role catalog in its own table, not in global_settings Five findings across three rounds were all the same choice. A set of live Postgres credentials was living in `global_settings`, which has generic read, list, write, config-export and CLI round-trip paths that know nothing about what they carry: the passwords reached the instance config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog, `GET /settings/global/{key}` and the settings listing returned them raw, and this round the redaction that fixed the last two turned `wmill instance push` into something that wipes every password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles` could also empty it outside the lock. The approved plan offered a table or `global_settings`, so this is the other option it already allowed rather than a new design. `datatable_role` is a table: no generic settings path can read it, list it, export it, write it or round-trip it, so none of the five needs a guard. The redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it. One row per role also removes the read-modify-write the concurrency work was about: two concurrent creates are two inserts, and the unique index on `name` is what settles a collision. The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to another transaction until commit, so without it both creates pass their `pg_roles` check. Also from this round: rename mappings are checked against the configuration they claim to describe, since fork pointers are rewritten from them — a caller could otherwise submit `main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning naming forks a delete stranded reached the response but not the screen: both the data table settings save and the workspace delete now show it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): validate a rename against the save it describes, and re-check under the locks Three from the round, all about deciding on state that could already have moved. A permission save resolved the data table and checked it was instance-backed before taking any lock, then wrote under one. A config save committing in between could move the table onto a PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in which case the write targeted a key that no longer existed and reported success having changed nothing. It now re-resolves and re-checks on the locked state. Rename validation checked that the source existed before and the target existed after, which still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows onto a different data table, silently, because it keeps resolving. The rule is now the actual old-to-new key transition — a source may only survive if another rename took its name, and a target may only pre-exist if another rename freed it. That also stops two sources sharing one target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so a swap cannot be done one save at a time, and refusing it was a regression against main. The pointer cascade now runs in two passes through a temporary name, the way the migration cascade one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once from what it named before the save. The tenant mutators say what they are for: they write an access decision for any workspace named, with an arbitrary mutation, and exist for the transaction that frees or renames a principal. Editing a decision on purpose belongs in the permissions endpoint. Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole document, so a workspace whose pointers name something else is not rewritten to a byte-identical value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from the JSON document, one of which would emit `pwd`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call `/workspaces/leave` — a different handler in a different crate with the same name — which deleted the membership and left `u/` in the tenant lists. Leaving and rejoining therefore restored the access the leave was supposed to end, and a later account taking the username would have inherited it. The regression test drives the route the client actually calls; without the fix it fails with "leaving kept the tenant". The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's admin connection, created `_wm_migrations` and read it before reaching the per-migration role check — so with nothing pending, nothing was checked at all. Rollback returned before its check when nothing was applied, and the status endpoint had none. All three now ask, before any connection is opened, whether the caller can reach the data table as any role at all; which role a given migration runs as is still decided per migration, and by the executor after that. Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now runs in the same transaction, so the drop, the row and every tenant list commit together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse to copy a data table that is under roles pg_dump carries no roles and the import runs with --no-privileges, so a copied data table arrives owned by the admin connection with no GRANT for any role. The settings clone brings `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were given, and are denied by Postgres on everything: an entry that reads as configured and answers nothing. Refuse the copy — in the import endpoint before any data moves, and in the fork path the CLI takes. Replaying the source's owners and ACLs into the clone is what lifts this, and is a change of its own. Dropping `permissions` from the copy instead would be the unsafe half, since the copy holds the parent's rows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse the clone's database too, not only its data A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only the second refused a data table under roles, so a fork asking to clone one created and registered an empty `wm_fork_…` instance database and then failed — and nothing collects it, since `drop_forked_datatable_databases` only drops entries carrying `forked_from` and no entry names this one. Refuse in both, so the clone stops before a database exists. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * nit worker error msg * fix pg_dump stuck on version 17 on nix * fix(datatables): refuse a malformed role annotation instead of ignoring it `-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed the annotation parser's exact-match rule, so the query fell through to the data table's default role and ran, silently, under a login the author did not choose. Naming a role exists precisely to not do that. A leading comment whose first word is `role` is now an annotation attempt: the keyword matches case-insensitively, one trailing `;` is tolerated, and anything else is an error naming the line. Only callers that already know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its comments. Also bumps the dev shell's postgres client to 18 — it trailed the server the dev database runs, which takes out every data table export, clone and fork-with-data. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse a malformed role query string instead of ignoring it `?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference parser's exact-match rule, so the connection resolved to the data table's default role and ran under a login the caller never asked for — the URI half of the same trap as a malformed `-- role` annotation. The key now matches case-insensitively, and anything else in the query string is an error naming it; `role` is the only parameter a reference takes. Callers that only need the entry keep a lenient `datatable_ref_name`, since they never act on the role. The DuckDB `ATTACH` parser propagates it rather than attaching under the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): carry the role annotation into the row_to_json retry The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment block never reached the second attempt — and with it the `-- role ` line that decides which login the query runs as. The retry connected as the data table's default role instead, so a query the first attempt was denied could succeed on the second, reported as "recovered with the row_to_json fix". Carry the leading comment block over. The retry itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore(datatables): don't mount the roles UI until the ACL editor lands Enforcement ships first. The permissions drawer is what turns roles on, and the catalog section is what creates them — both are only useful once there is a way to grant a role the privileges it needs, which arrives with the ACL editor. Left mounted they would offer a feature whose other half does not exist. The two components are complete and reviewed; only their call sites here are commented out, with a note pointing the follow-up PRs at them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): honour `-- role: x`, and fix the DuckDB attach test Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse to roll back the catalog while roles exist The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * refactor(datatables): settle clonability in one place, before anything is created A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): let a retried clone reclaim its own leftover database A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) * Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bounce the streams reading a data table when it is deleted Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) * refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. * fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * feat(datatables): Ducklake catalogs on the external instance cluster Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): never grant CREATEROLE to custom_instance_user on the external cluster Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): state the authorization contract of external database usage lookups Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): drop a DuckDB data table secret once its ATTACH has used it Co-Authored-By: Claude Opus 5 (1M context) * perf(datatables): resolve a workspace's data tables per pointer hop, not per entry Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): hold the parent's settings while a fork points at its data tables Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse repointing the external cluster while it is in use, and keep verify-ca working for pg_dump Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): protect external databases pending fork cleanup, and describe Ducklake usage in the API Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): per-cluster data table role catalogs, with roles on the external instance cluster Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): write the external cluster setting under the lifecycle lock, and check fork targets are registered Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): register external fork catalogs under the lifecycle lock, and keep certificate verification in DuckDB attaches Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep certificate verification when DuckDB attaches an external data table Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep certificate verification when DuckDB attaches an external data table Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse fork cleanup of an external database another workspace uses Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse rolling back while external data tables are under roles, and type external_instance in the CLI Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): stop counting storage-only fork cleanup rows as uses of an external database Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bind fork database copies to their workspace, and count every use before dropping one Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): authenticate instance database setup before writing its status, and keep a fork reservation across it Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): create external databases only on a cluster setup succeeded on, and document the registry reader Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep only the most recently used DuckDB root certificate files Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): migrate fork reservations on workspace rename, and lock the parent's data tables for the whole fork Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): take the fork data table lock once, before the external cluster's Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): describe the external instance cluster and how to run one locally Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep fork reservations private, drop a cleaned-up entry with its database, and serialize cleanup with settings saves Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): hold the fork lock across a fork import, and carry the reservation inside the setup write Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): check the external cluster setting on its own transaction, and gate the registry probe Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * chore: update ee-repo-ref to 7e338e4dabf91689bfd7fb0333c6534040b17b59 This commit updates the EE repository reference after PR #787 was merged in windmill-ee-private. Previous ee-repo-ref: 0edd40979cf36bfba59323f3f6a0811ae1369cf5 New ee-repo-ref: 7e338e4dabf91689bfd7fb0333c6534040b17b59 Automated by sync-ee-ref workflow. * fix(datatables): keep DuckDB root certificate files in the job directory Co-Authored-By: Claude Opus 5 (1M context) * chore: drop the unused json import from the settings crate Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): drop the serde_json::json import left unused by the fork setup write Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): accept external_instance data tables in the settings form type Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize fork database reservations, cleanup, setup and Ducklake saves on the database - A fork copy is created and registered under its workspace's fork lock, and refused once the workspace is archived; a rename re-migrates reservations under that lock after archiving. - Ducklake saves lock every instance database they newly name, as data table saves do. - Instance database setup holds the database's lock until its entry is written. - Fork import also holds the database's lock across the restore. - Cleanup re-reads the entry under its locks before dropping anything. - Non-superadmins no longer see fork copies reserved for workspaces they are not in. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): state the authorization contract of the external cluster status and write check Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): lock the fork copy at finalization, and order cleanup's registry write after its settings row - Fork finalization holds the copy's database lock from its availability check to the commit. - Cleanup removes the registry entry in its own transaction, in a task of its own, instead of on a second connection; a rename migrates reservations after its settings rewrites, so both take the settings rows before the registry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): hold the fork lock across a rename's settings copy Fork cleanup of the old id could otherwise drop a copy the renamed workspace goes on using. Also states the authorization contract of the instance database drop helpers. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse moving the external cluster to another host or port while it holds databases Every settings writer goes through the same check as removal: the login, TLS and maintenance database may still change, the cluster may not. Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): clone a data table under roles with its owners and grants (#11120) Claude-Session: https://claude.ai/code/session_01UbrtwiYNfayrmqouBJHwGV Co-authored-by: Claude Opus 5 (1M context) * fix: open the raw app data table drawer when the workspace has none Selecting the first data table of an empty list passed undefined to the name check, which threw instead of opening the drawer on no data table. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): offer cloning a data table under roles where its grants can be replayed The server clones such a data table and replays the source's owners and grants, which only the Enterprise Edition does, so the fork wizard hid both clone options everywhere instead of on a build that cannot replay them. Co-Authored-By: Claude Opus 5 (1M context) * docs: name the placeholder the empty raw app data drawer renders Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): replace a job directory file at the DuckDB root certificate path instead of trusting it Co-Authored-By: Claude Opus 5 (1M context) * test: pin the enterprise refusal the role pickers read as 'not under roles' The server's sentence and the frontend's copy of it were coupled by nothing, so rewording either one turned every role picker on a community build into a failed lookup. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): type Ducklake catalogs on the external instance cluster in the settings form Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): read a roles answer only for the workspace it was asked in A fork and its parent each have their own roles on a data table of the same name, so an answer stamped with the name alone settled the role from the workspace the editor was acting on before. Also derive the AI table creation flag from the data replaced into the editor: data naming no data table left the flag on from before. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep an instance database a settings save is waiting to name Cleanup for a database whose setup failed took the lock first, read no user, and dropped it while a save blocked on that same lock was about to commit a reference to it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): check the workspace stamp in the default database selector too Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): read what a save racing instance-database cleanup committed A transaction blocked on the lock may still roll back, so keeping the database for it stranded one whose name then blocks every retry: it is let through and its outcome read instead. The waiter query also matches this database's locks only, since pg_locks spans the cluster. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): tell a waiting request apart from the workspaces using a database Both callers render what cleanup returns as the workspaces that keep the database, so a waiting request's pid read as one of them. Each now words that case itself, and the give-up comment names where the kept name actually goes. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): take the cleanup lock on a connection the pool cannot reclaim A session lock outlives the future holding it, so a cancellation between taking it and releasing it handed a locked session back to the pool, where every later settings save waits on it. Detached, the connection closes when it is dropped and the server releases the lock with the session. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): close the cleanup connection on drop instead of detaching it Detaching released the pool permit while the session stayed alive, so concurrent cleanups waiting on their locks could open as many connections as they liked. Closing on drop covers the same cancellation and keeps them counted. Co-Authored-By: Claude Opus 5 (1M context) * chore: update ee-repo-ref to 179d2454a063ee818a9387a1eafdd354a217a16c This commit updates the EE repository reference after PR #802 was merged in windmill-ee-private. Previous ee-repo-ref: c2e43f5b5ff753d339b70e232fd17b6ffecb054d New ee-repo-ref: 179d2454a063ee818a9387a1eafdd354a217a16c Automated by sync-ee-ref workflow. * chore: update ee-repo-ref to fd5b8af748f2c985b13e18d9ea30894f3bd7e9a3 This commit updates the EE repository reference after PR #798 was merged in windmill-ee-private. Previous ee-repo-ref: 3145e422d61d580f0a82804f075285c112879da0 New ee-repo-ref: fd5b8af748f2c985b13e18d9ea30894f3bd7e9a3 Automated by sync-ee-ref workflow. * fix(datatables): classify a fork import's target from the resolution that built its connection A second read could see the entry flipped to a resource and skip the reservation check while the connection already built still reached the instance cluster. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): run external database lifecycle on the caller's transaction, one lock order for fork cleanup Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): check a fork copy's reservation on the locked transaction's connection Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): make and drop fork copies of external data tables on the external cluster Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a fork whose external copy something already names Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): check an external fork copy's uses before removing its entry, so child fork pointers still count Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): migrate external cluster fork reservations on workspace rename Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): configure the external instance cluster and pick its databases from the UI Instance settings gets an External Postgres tab: the cluster's connection, the setup run with its report, and the databases Windmill created there. Data table and Ducklake settings offer the external_instance kind, with a picker over those databases. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the external cluster form out of the instance settings the page bulk-saves The form seeded its key into the settings store after the page snapshotted them, so merely opening the tab made the page send a setting the validator refuses, failing an admin's unrelated save. The form is local state now, and only the setup writes the key. The database picker also treats a superadmin-only listing as authoritative: a workspace admin, who cannot list, no longer sees a saved database as missing. Co-Authored-By: Claude Opus 5 (1M context) * fix(tests): app paths are validated, so a guest app cannot be renamed to one with a space Path validation on apps landed after this test, which still expected a space to be accepted and created its fixture on a ':' path. The scopable-path guard it exists for is kept by planting that path on the row, which is now the only way an app can hold one. Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): one Managed Postgres tab, and a switch for Windmill's own database The instance settings tab covers both substrates Windmill administers: the external cluster, which can now be disabled once nothing sits on it, and Windmill's own database, which an operator can turn off so the cluster is the only one a workspace may newly name. A save that names it then refuses, as the create endpoint does. Workspace settings name them the way an admin meets them: Postgres Resource, and Managed instance, qualified as Internal or External only while both are on offer. Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): offer the external cluster in the add-data-table wizard, and give Ducklake its room back The wizard gets the external cluster as a fourth substrate: pick a database it already holds or name a new one, which the run creates before writing the entry, so Try again does not trip on a database the last attempt made. Ducklake's maintenance column moves into the settings popover next to the extra args, which the wider catalog select had squeezed the name box out of. A setting a component saves itself now also moves the page's baseline, or Discard would restore what it replaced and a later save would send that stale value back. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): only a database this wizard run made makes its retry idempotent A retry skipped the create whenever the name was registered, which let a run adopt a database another superadmin had made and share their data without the warning the existing-database branch shows. The run tracks what it created instead, and reports it so Discard can say the database is still there. Creating one now refreshes the registry the next run's default name and its validation read, and that default counts the external cluster's databases rather than the instance's. Row ownership carries the substrate: "instance" and "external_instance" can hold the same name, and a row repointed between them while a run probes must not read as that run's own. Also fixes the merge leftovers in the DuckDB executor's tests, which cargo check never compiles: the new PgDatabase field and the attach helper's job directory. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): shorten the picked managed kind, and say what an empty database list means The closed select has room for the qualifier, not the whole name, so a picked managed kind reads "Managed (Internal)". An external entry gets the tooltip its internal counterpart has, and a database picker with nothing in it says to type a name rather than "No items found". Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the wizard's external-database bookkeeping optional and parked The run deps gained a required field, which every existing caller of runSetup -- the sibling test suite included -- does not pass. It defaults to empty instead, the parked payload carries it across a Supabase redirect as it claims to, and a resumed run counts it as something left behind. Two regressions pin the create: made on a first attempt, skipped only for a database this run made. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): review an external-cluster database as one, not as a Postgres resource The review step fell through to the resource branch for the new provider, so it announced a connection already in the workspace for a database that does not exist yet. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): show the server's refusal instead of an object in the toast Saving data table settings passed the whole error to the toast, which rendered the request and response as JSON, and the instance-database wizard replaced it with "check console". Both surface the message the API sent, through the helper that exists for it. Co-Authored-By: Claude Opus 5 (1M context) * chore: point at the EE commit dropping the unused import Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): define a role on the cluster its data table sits on The permissions drawer created and listed roles without naming a cluster, so a role defined from an external data table became a login on Windmill's own, and the refresh then replaced the correct catalog with the internal one. The drawer passes the data table's cluster to both. The wizard also read a database it had just created as a name collision, which blocked returning to the failed attempt and finishing under the same name. And the merge had left the CONNECT-grant pass and its log in both the caller and the callee. Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): manage the external cluster's role catalog, and name the cluster in the copy The roles drawer offered one catalog, so the external cluster's roles could only be reached through a data table sitting on it — and became unreachable once the last one was gone, while the cluster could not be unset until they were dropped. Opened from the page it now offers the clusters the instance has; a data table's own drawer still pins its cluster, and its copy names that cluster rather than "the instance". A DuckDB attach also decides whether to verify certificates the way every other Postgres connection does, so a resource carrying a root certificate is no longer downgraded to require here alone. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bind the roles list to the cluster it was read for Switching catalogs left the previous cluster's rows on screen and applied whichever response landed last, so a slower read could seat one cluster's logins under the other's heading while every control acted on the wrong id — invisible where a name exists on both. The switch clears the rows, a token discards a response the selection has moved past, and the controls stay inert while a catalog is being read. The drop confirmation also names the cluster whose databases it reaches. Co-Authored-By: Claude Opus 5 (1M context) * chore: update ee-repo-ref to dfe7b8b9204d21e0264fbea1c6f6eedf9e738d56 This commit updates the EE repository reference after PR #810 was merged in windmill-ee-private. Previous ee-repo-ref: 30db1b33bda0446f5fc5dfe353fbb226d57a26d4 New ee-repo-ref: dfe7b8b9204d21e0264fbea1c6f6eedf9e738d56 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] Co-authored-by: Ruben Fiszel --- AGENTS.md | 4 + ...d368347c59f36fd87df6dc7996152ccb84af0.json | 38 -- ...9ad2db940c2455ab19cc95e5198baf96d5629.json | 29 - ...df2a63695f8cbf8af648da5a0a77a5b9d02ba.json | 17 - ...12373638be391f884f81dc387ffc465badac6.json | 20 - ...8dc45020a2a553d8874c49f9eafedea5a9d40.json | 24 - backend/Cargo.lock | 1 + backend/Cargo.toml | 1 + backend/ee-repo-ref.txt | 2 +- ...0917133824_datatable_role_cluster.down.sql | 22 + ...260917133824_datatable_role_cluster.up.sql | 8 + backend/tests/instance_config.rs | 46 ++ .../tests/datatable_roles.rs | 12 +- backend/windmill-api-settings/src/lib.rs | 249 +++++++- .../src/datatable_acl.rs | 78 ++- .../src/datatable_clone.rs | 100 +++- .../src/datatable_migrations.rs | 6 +- .../windmill-api-workspaces/src/workspaces.rs | 533 ++++++++++++++---- .../src/workspaces_extra.rs | 184 +++++- backend/windmill-api/openapi.yaml | 177 +++++- backend/windmill-common/Cargo.toml | 1 + .../windmill-common/src/datatable_roles.rs | 171 ++++-- .../src/datatable_roles_oss.rs | 64 ++- .../src/external_instance_pg.rs | 456 +++++++++++++++ .../src/external_instance_pg_oss.rs | 82 +++ .../windmill-common/src/global_settings.rs | 8 + .../windmill-common/src/instance_config.rs | 103 ++-- backend/windmill-common/src/lib.rs | 153 ++++- backend/windmill-common/src/workspaces.rs | 197 ++++++- .../windmill-worker/src/duckdb_executor.rs | 285 ++++++++-- cli/src/commands/datatable/datatable.ts | 6 +- docs/external-instance-datatables.md | 88 +++ .../src/lib/components/InstanceSetting.svelte | 22 +- .../lib/components/InstanceSettings.svelte | 13 + .../src/lib/components/instanceSettings.ts | 31 + .../ExternalInstancePgSettings.svelte | 471 ++++++++++++++++ .../InstancePgSettings.svelte | 65 +++ .../AddDataTableWizard.svelte | 153 ++++- .../CustomInstanceDbWizardModal.svelte | 5 +- .../DataTablePermissionsButton.svelte | 20 +- .../DataTableRolesSection.svelte | 71 ++- .../DataTableSettings.svelte | 137 ++++- .../workspaceSettings/DucklakeSettings.svelte | 399 +++++++------ .../ExternalInstanceDbSelect.svelte | 117 ++++ .../InstanceRolesButton.svelte | 8 +- .../addDataTableModel.test.ts | 48 +- .../workspaceSettings/addDataTableModel.ts | 99 +++- .../workspaceSettings/datatableTelemetry.ts | 2 +- .../workspaceSettings/utils.svelte.ts | 24 + .../workspaceSettings/wizardParking.ts | 3 + 50 files changed, 4121 insertions(+), 732 deletions(-) delete mode 100644 backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json delete mode 100644 backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json delete mode 100644 backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json delete mode 100644 backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json delete mode 100644 backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json create mode 100644 backend/migrations/20260917133824_datatable_role_cluster.down.sql create mode 100644 backend/migrations/20260917133824_datatable_role_cluster.up.sql create mode 100644 backend/windmill-common/src/external_instance_pg.rs create mode 100644 backend/windmill-common/src/external_instance_pg_oss.rs create mode 100644 docs/external-instance-datatables.md create mode 100644 frontend/src/lib/components/instanceSettings/ExternalInstancePgSettings.svelte create mode 100644 frontend/src/lib/components/instanceSettings/InstancePgSettings.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/ExternalInstanceDbSelect.svelte diff --git a/AGENTS.md b/AGENTS.md index 858e9329ff..850538f080 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,10 @@ Open-source platform for internal tools, workflows, API integrations, background - **Agent workers**: `docs/agent-worker-e2e.md` — building and running one locally. An agent reaches the DB only through the API, so `Connection::Http` paths are never taken by a plain `cargo run`; a normal build cannot start one at all. +- **External instance data tables**: `docs/external-instance-datatables.md` — the cluster Windmill + administers behind `external_instance` data tables and Ducklake catalogs: its invariants (one + lifecycle lock, managed-object markers, the setup gate, per-cluster roles, fork copy ownership) + and how to run one locally - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option` diff --git a/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json b/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json deleted file mode 100644 index a03fab7081..0000000000 --- a/backend/.sqlx/query-71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT id, name, enabled, pwd FROM datatable_role", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "pwd", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true - ] - }, - "hash": "71ee2cb6661cca1fa4d8874a7f6d368347c59f36fd87df6dc7996152ccb84af0" -} diff --git a/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json b/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json deleted file mode 100644 index c36b6fb361..0000000000 --- a/backend/.sqlx/query-79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT ws.workspace_id AS \"workspace_id!\", dt.key AS \"datatable!\"\n FROM workspace_settings ws\n JOIN workspace w ON w.id = ws.workspace_id AND w.deleted = false\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id <> $1\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $2\n ORDER BY ws.workspace_id, dt.key\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "datatable!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "79799b5a2e499df6c28e286c42b9ad2db940c2455ab19cc95e5198baf96d5629" -} diff --git a/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json b/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json deleted file mode 100644 index 7627d9828d..0000000000 --- a/backend/.sqlx/query-86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO datatable_role (id, name, enabled, pwd) VALUES ($1, $2, $3, $4)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Text" - ] - }, - "nullable": [] - }, - "hash": "86af9d51a158ea5cb6161461ecddf2a63695f8cbf8af648da5a0a77a5b9d02ba" -} diff --git a/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json b/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json deleted file mode 100644 index 4b3a2f33e9..0000000000 --- a/backend/.sqlx/query-b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT jsonb_object_keys(value->'databases') FROM global_settings\n WHERE name = 'custom_instance_pg_databases'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jsonb_object_keys", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "b9842d2d8abf382bd82d8fa1de012373638be391f884f81dc387ffc465badac6" -} diff --git a/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json b/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json deleted file mode 100644 index 49875acd78..0000000000 --- a/backend/.sqlx/query-d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT dt.key AS \"datatable!\"\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt\n WHERE ws.workspace_id = $1\n AND dt.key <> $2\n AND NOT dt.value ? 'permissions'\n AND dt.value->'database'->>'resource_type' = 'instance'\n AND dt.value->'database'->>'resource_path' = $3\n ORDER BY dt.key", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "datatable!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d48ca62c86b1af7a9dd2450c1c28dc45020a2a553d8874c49f9eafedea5a9d40" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9182843789..e80d0edf08 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15660,6 +15660,7 @@ dependencies = [ "pin-project-lite", "pkcs1", "postgres-native-tls 0.5.3", + "postgres-protocol", "prometheus", "quick_cache", "rand 0.9.5", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 77f512a3dd..8571b0b99c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -626,6 +626,7 @@ wasm-bindgen-test = "^0" convert_case = "0.6.0" getrandom = "0.2" tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]} +postgres-protocol = "0.6" rust-postgres = { package = "tokio-postgres", git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe"} rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/MaterializeInc/rust-postgres", features = ["runtime"], rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" } bit-vec = "=0.6.3" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 58f9aaf07d..21e445394e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9855e1b7a43a0a33e04f8accf1c497af3fd9b139 +dfe7b8b9204d21e0264fbea1c6f6eedf9e738d56 diff --git a/backend/migrations/20260917133824_datatable_role_cluster.down.sql b/backend/migrations/20260917133824_datatable_role_cluster.down.sql new file mode 100644 index 0000000000..379f1e3d34 --- /dev/null +++ b/backend/migrations/20260917133824_datatable_role_cluster.down.sql @@ -0,0 +1,22 @@ +-- Roles on the external cluster are live logins there; dropping the column would forget them. +LOCK TABLE datatable_role; +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM datatable_role WHERE cluster <> 'instance') THEN + RAISE EXCEPTION 'datatable_role holds roles on the external instance cluster. Delete them in instance settings first.'; + END IF; + -- Before this, only data tables on Windmill's own cluster could be under roles, and a role + -- block left with just `admin` survives deleting every external role. + IF EXISTS ( + SELECT 1 FROM workspace_settings ws, + jsonb_each(CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt + WHERE dt.value->'database'->>'resource_type' = 'external_instance' + AND dt.value ? 'permissions' + ) THEN + RAISE EXCEPTION 'external instance data tables are still under roles. Turn their roles off first.'; + END IF; +END $$; +ALTER TABLE datatable_role DROP CONSTRAINT datatable_role_cluster_name_key; +ALTER TABLE datatable_role ADD CONSTRAINT datatable_role_name_key UNIQUE (name); +ALTER TABLE datatable_role DROP COLUMN cluster; diff --git a/backend/migrations/20260917133824_datatable_role_cluster.up.sql b/backend/migrations/20260917133824_datatable_role_cluster.up.sql new file mode 100644 index 0000000000..d63f06f1f4 --- /dev/null +++ b/backend/migrations/20260917133824_datatable_role_cluster.up.sql @@ -0,0 +1,8 @@ +-- A data table role is a Postgres login on one cluster: Windmill's own ('instance'), or the external +-- instance cluster ('external_instance'). Role names are the cluster's own key, so they are unique +-- per cluster rather than across the instance. +ALTER TABLE datatable_role + ADD COLUMN cluster VARCHAR(20) NOT NULL DEFAULT 'instance' + CHECK (cluster IN ('instance', 'external_instance')); +ALTER TABLE datatable_role DROP CONSTRAINT datatable_role_name_key; +ALTER TABLE datatable_role ADD CONSTRAINT datatable_role_cluster_name_key UNIQUE (cluster, name); diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index 26a46d272e..52ebbee7e7 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1609,6 +1609,52 @@ async fn declarative_sync_rejects_an_unusable_default_allowed_origins(db: Pool

    ) { + clear_settings_and_configs(&db).await; + let cluster = |host: &str, password: &str| serde_json::json!({ "host": host, "port": 5432, "user": "wm_admin", "password": password }); + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES + ('external_instance_pg', $1), + ('external_instance_pg_state', '{\"databases\": {\"dt_a\": {\"success\": true}}}')", + ) + .bind(cluster("pg-a.internal", "one")) + .execute(&db) + .await + .unwrap(); + let current = BTreeMap::from([( + "external_instance_pg".to_string(), + cluster("pg-a.internal", "one"), + )]); + let sync = |value: serde_json::Value| { + let desired = BTreeMap::from([("external_instance_pg".to_string(), value)]); + let (db, current) = (db.clone(), current.clone()); + async move { + windmill_common::instance_config::sync_global_settings_declarative( + &db, ¤t, &desired, + ) + .await + } + }; + + let err = sync(cluster("pg-b.internal", "one")) + .await + .expect_err("another host must be refused while dt_a is registered"); + assert!(err.to_string().contains("dt_a"), "got: {err}"); + assert_eq!( + get_global_setting(&db, "external_instance_pg").await, + Some(cluster("pg-a.internal", "one")) + ); + + sync(cluster("PG-A.internal ", "two")) + .await + .expect("a new login on the same cluster must sync"); +} + /// The accent color is interpolated into a stylesheet every user loads, so the operator /// path must refuse anything but `#rrggbb` just like the settings API does. #[sqlx::test(fixtures("base"))] diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index 4948a0098a..ddb94f8c2b 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -391,7 +391,11 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R assert_eq!(a.0, 200, "{}", a.1); assert_eq!(b.0, 200, "{}", b.1); - let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let catalog = windmill_common::datatable_roles::read_role_catalog( + &db, + windmill_common::datatable_roles::DatatableRoleCluster::Instance, + ) + .await?; let recorded: Vec<&str> = catalog.values().map(|r| r.name.as_str()).collect(); for name in &names { assert!( @@ -460,7 +464,11 @@ async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( let body = resp.text().await?; assert_eq!(status, 400, "{body}"); - let catalog = windmill_common::datatable_roles::read_role_catalog(&db).await?; + let catalog = windmill_common::datatable_roles::read_role_catalog( + &db, + windmill_common::datatable_roles::DatatableRoleCluster::Instance, + ) + .await?; let role = catalog .get(&id) .expect("a failed delete keeps the entry to retry"); diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index ca93d3c682..71cbcc8460 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -42,7 +42,6 @@ use axum::{ routing::{get, post}, Json, Router, }; -use serde_json::json; use serde::{Deserialize, Serialize}; use windmill_ai::ai_cache::bump_instance_ai_config_revision; @@ -61,6 +60,7 @@ use windmill_common::{ ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, + EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -181,6 +181,22 @@ pub fn global_service() -> Router { "/refresh_custom_instance_user_pwd", post(refresh_custom_instance_user_pwd), ) + .route( + "/external_instance_pg/status", + get(get_external_instance_pg_status), + ) + .route( + "/external_instance_pg/setup", + post(setup_external_instance_pg), + ) + .route( + "/external_instance_pg/databases", + get(list_external_instance_pg_databases), + ) + .route( + "/external_instance_pg/databases/{name}", + post(create_external_instance_pg_database).delete(drop_external_instance_pg_database), + ) .route( "/setup_custom_instance_pg_database/{name}", post(setup_custom_instance_pg_database), @@ -853,6 +869,14 @@ pub async fn set_global_setting_internal( ))); } + if key == EXTERNAL_INSTANCE_PG_SETTING { + return windmill_common::external_instance_pg::write_external_instance_pg_setting( + db, + Some(&value), + ) + .await; + } + run_setting_pre_write_hook(db, &key, &value).await?; match value { @@ -1248,7 +1272,7 @@ async fn set_instance_config( let desired_map = desired.global_settings.to_settings_map(); if !desired_map.is_empty() { let current_map = current.global_settings.to_settings_map(); - let settings_diff = + let mut settings_diff = instance_config::diff_global_settings(¤t_map, &desired_map, ApplyMode::Merge); let ai_config_changed = settings_diff .upserts @@ -1277,8 +1301,15 @@ async fn set_instance_config( } for (key, value) in &settings_diff.upserts { - run_setting_pre_write_hook(&db, key, value).await?; + if key != EXTERNAL_INSTANCE_PG_SETTING { + run_setting_pre_write_hook(&db, key, value).await?; + } } + windmill_common::external_instance_pg::write_external_instance_pg_from_diff( + &db, + &mut settings_diff, + ) + .await?; instance_config::apply_settings_diff(&db, &settings_diff) .await @@ -1643,6 +1674,8 @@ struct CustomInstanceDb { tag: Option, #[serde(default, skip_serializing_if = "Vec::is_empty")] used_by_workspaces: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + workspace_id: Option, } #[derive(Deserialize, Debug, Serialize, Default)] @@ -1685,7 +1718,40 @@ async fn list_custom_instance_pg_databases( )) })?; - if windmill_api_auth::is_super_admin_authed(&db, &authed).await? { + if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? { + // A fork copy's name gives away the workspace it was reserved for, so every pending fork on + // the instance would be listed. Kept for members of that workspace, and wherever the + // caller's workspaces use it, e.g. the fork it was finalized into. + let reserved_visible: BTreeSet = sqlx::query_scalar( + r#"SELECT e.k FROM global_settings gs + CROSS JOIN LATERAL jsonb_each(gs.value->'databases') AS e(k, v) + WHERE gs.name = 'custom_instance_pg_databases' AND e.v->>'workspace_id' IS NOT NULL + AND (EXISTS (SELECT 1 FROM usr WHERE usr.email = $1 + AND usr.workspace_id = e.v->>'workspace_id') + OR EXISTS (SELECT 1 FROM usr JOIN workspace_settings ws + ON ws.workspace_id = usr.workspace_id + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt + WHERE usr.email = $1 + AND dt.value->'database'->>'resource_type' = 'instance' + AND dt.value->'database'->>'resource_path' = e.k))"#, + ) + .bind(&authed.email) + .fetch_all(&db) + .await? + .into_iter() + .collect(); + result.retain(|dbname, entry| { + entry.workspace_id.is_none() || reserved_visible.contains(dbname) + }); + // Which workspace reserved a copy is still only for superadmins. + for entry in result.values_mut() { + entry.workspace_id = None; + } + return Ok(Json(result)); + } + { // Enrich each database with the list of workspaces referencing it through // either a ducklake catalog or a datatable database whose resource_type is // 'instance'. Not stored in DB to avoid drift. @@ -1741,6 +1807,139 @@ async fn refresh_custom_instance_user_pwd( Ok(Json(())) } +async fn get_external_instance_pg_status( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult { + require_super_admin(&db, &authed).await?; + Ok(Json( + windmill_common::external_instance_pg::external_instance_pg_status(&db).await?, + )) +} + +#[derive(Deserialize)] +struct SetupExternalInstancePgBody { + #[serde(default)] + rotate_passwords: bool, +} + +async fn setup_external_instance_pg( + authed: ApiAuthed, + Extension(db): Extension, + Json(body): Json, +) -> JsonResult { + require_super_admin(&db, &authed).await?; + let report = windmill_common::external_instance_pg::setup_external_instance_pg_unchecked( + &db, + body.rotate_passwords, + ) + .await?; + let rotated = body.rotate_passwords.to_string(); + let success = report.success.to_string(); + windmill_audit::audit_oss::audit_log( + &db, + &authed, + "settings.setup_external_instance_pg", + windmill_audit::ActionKind::Update, + "global", + Some(&authed.email), + Some( + [ + ("rotate_passwords", rotated.as_str()), + ("success", success.as_str()), + ] + .into(), + ), + ) + .await?; + Ok(Json(report)) +} + +#[derive(Serialize)] +struct ExternalInstancePgDatabase { + #[serde(flatten)] + status: windmill_common::instance_config::CustomInstanceDb, + used_by_workspaces: Vec, +} + +async fn list_external_instance_pg_databases( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult> { + require_super_admin(&db, &authed).await?; + let databases = windmill_common::external_instance_pg::external_instance_databases(&db).await?; + let mut usages = + windmill_common::external_instance_pg::external_instance_database_usages(&db).await?; + Ok(Json( + databases + .into_iter() + .map(|(name, status)| { + let used_by_workspaces = usages.remove(&name).unwrap_or_default(); + ( + name, + ExternalInstancePgDatabase { + status, + used_by_workspaces: used_by_workspaces.into_iter().collect(), + }, + ) + }) + .collect(), + )) +} + +async fn create_external_instance_pg_database( + authed: ApiAuthed, + Extension(db): Extension, + Path(dbname): Path, + Json(body): Json, +) -> JsonResult<()> { + require_super_admin(&db, &authed).await?; + let tag = body.tag.as_deref().unwrap_or("datatable"); + let mut tx = db.begin().await?; + windmill_common::external_instance_pg::create_external_instance_database_unchecked( + &db, &mut tx, &dbname, tag, None, + ) + .await?; + tx.commit().await?; + windmill_audit::audit_oss::audit_log( + &db, + &authed, + "settings.create_external_instance_pg_database", + windmill_audit::ActionKind::Create, + "global", + Some(&authed.email), + Some([("dbname", dbname.as_str()), ("tag", tag)].into()), + ) + .await?; + Ok(Json(())) +} + +async fn drop_external_instance_pg_database( + authed: ApiAuthed, + Extension(db): Extension, + Path(dbname): Path, +) -> JsonResult<()> { + require_super_admin(&db, &authed).await?; + // A data table naming a dropped database fails on every job, far from the drop that caused it. + let mut tx = db.begin().await?; + windmill_common::external_instance_pg::drop_external_instance_database_unchecked( + &mut tx, &dbname, None, + ) + .await?; + tx.commit().await?; + windmill_audit::audit_oss::audit_log( + &db, + &authed, + "settings.drop_external_instance_pg_database", + windmill_audit::ActionKind::Delete, + "global", + Some(&authed.email), + Some([("dbname", dbname.as_str())].into()), + ) + .await?; + Ok(Json(())) +} + #[derive(Deserialize)] struct SetupCustomInstanceDbBody { tag: Option, @@ -1752,18 +1951,46 @@ async fn setup_custom_instance_pg_database( Path(dbname): Path, Json(body): Json, ) -> JsonResult { + // Before anything is recorded: the status written below replaces the registry entry, and with it + // the workspace a fork copy is reserved for. + require_super_admin(&db, &authed).await?; + windmill_common::workspaces::ensure_instance_pg_available(&db).await?; + // Fork cleanup checks and drops the database and its entry under this lock. Held from before + // the setup creates the database to after its entry is written, neither lands on the other's + // half-done state: a dropped database with its entry written back, or the reverse. + let mut tx = db.begin().await?; + windmill_common::datatable_roles::lock_instance_databases_governance(&mut tx, [dbname.trim()]) + .await?; let mut logs = CustomInstanceDbLogs::default(); let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await; let success = result.is_ok(); let error = result.err().map(|e| e.to_string()); - let status = - CustomInstanceDb { logs, success, error, tag: body.tag, used_by_workspaces: vec![] }; + let status = CustomInstanceDb { + logs, + success, + error, + tag: body.tag, + used_by_workspaces: vec![], + workspace_id: None, + }; let status_json = serde_json::to_value(&status).map_err(to_anyhow)?; - // Save that the database was setup successfully - sqlx::query!( - r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#, - json!({ dbname: status_json }) - ).execute(&db).await?; + // The fork reservation is carried over inside the write, from whatever the row holds then: a + // rename migrating it while the setup above ran would otherwise be overwritten with the value + // this request started from, stranding the copy under the archived workspace. + let saved = sqlx::query_scalar::<_, serde_json::Value>( + r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', + COALESCE(value->'databases', '{}'::jsonb) + || jsonb_build_object($1::text, $2::jsonb || jsonb_build_object( + 'workspace_id', value->'databases'->$1::text->'workspace_id'))) + WHERE name = 'custom_instance_pg_databases' + RETURNING value->'databases'->$1::text"#, + ) + .bind(&dbname) + .bind(&status_json) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + let status: CustomInstanceDb = serde_json::from_value(saved).map_err(to_anyhow)?; Ok(Json(status)) } diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index 04dc3d4e80..ebab6214ce 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -//! Ownership and grants on the objects of an instance data table. +//! Ownership and grants on the objects of a data table on a cluster Windmill manages. //! //! [`datatable_permissions`](crate::datatable_permissions) decides who may connect as which role; //! this decides what each role may then touch. Every change is a real `GRANT`, `REVOKE`, @@ -33,7 +33,7 @@ use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::datatable_roles::{ lock_role_catalog, quote_ident, read_role_catalog, read_role_catalog_tx, DatatableRoleCatalog, - ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, + DatatableRoleCluster, ADMIN_DATATABLE_ROLE, CUSTOM_INSTANCE_USER, }; use windmill_common::error::{pg_error_message, Error, JsonResult, Result}; use windmill_common::workspaces::{resolve_governing_datatable, DataTable, GoverningDatatable}; @@ -297,22 +297,22 @@ fn role_names(catalog: &DatatableRoleCatalog) -> Vec { names } -fn ensure_instance(governing: &GoverningDatatable) -> Result<()> { - if governing.is_instance() { - return Ok(()); - } - Err(Error::BadRequest(format!( - "Data table '{}' is backed by a Postgres resource, so its access is managed on that \ - server directly. Only a data table on the Windmill instance's own database has data \ - table roles to grant to.", - governing.name - ))) +/// The cluster whose roles the data table's grants name. +fn ensure_managed(governing: &GoverningDatatable) -> Result { + governing.role_cluster().ok_or_else(|| { + Error::BadRequest(format!( + "Data table '{}' is backed by a Postgres resource, so its access is managed on that \ + server directly. Only a data table on a database Windmill manages has data table \ + roles to grant to.", + governing.name + )) + }) } /// The data table's `admin` connection, and the notices Postgres sends on it. /// -/// Authorization: connects as `custom_instance_user` with the instance's own credentials and checks -/// nothing. Callers MUST have authorized the request first — a request about to be refused must +/// Authorization: connects as `custom_instance_user` with the cluster's stored credentials and +/// checks nothing. Callers MUST have authorized the request first — a request about to be refused must /// not get as far as this connection. async fn connect_as_admin_unchecked( db: &DB, @@ -322,21 +322,33 @@ async fn connect_as_admin_unchecked( mpsc::UnboundedReceiver, String, )> { - ensure_instance(governing)?; + let cluster = ensure_managed(governing)?; // Built from the authorized entry, never by resolving the settings again: a save in between // could point the entry at a resource on another server and back, and this connection would // then alter a database the later checks of the entry never see. - let mut pg = PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; - pg.dbname = governing + let dbname = governing .datatable .database .as_ref() .expect("a governing entry owns a database") .resource_path .clone(); - pg.user = Some(CUSTOM_INSTANCE_USER.to_string()); - pg.password = Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); - let dbname = pg.dbname.clone(); + let pg = match cluster { + DatatableRoleCluster::Instance => { + let mut pg = + PgDatabase::parse_uri(&windmill_common::get_database_url().await?.as_str().await)?; + pg.dbname = dbname.clone(); + pg.user = Some(CUSTOM_INSTANCE_USER.to_string()); + pg.password = Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); + pg + } + DatatableRoleCluster::ExternalInstance => { + windmill_common::external_instance_pg::external_instance_connection_unchecked( + db, &dbname, false, + ) + .await? + } + }; let (client, notices) = connect_with_notices(db, &pg).await?; Ok((client, notices, dbname)) } @@ -1085,12 +1097,12 @@ async fn get_datatable_acl( let target: AclTarget = query.try_into()?; let governing = resolve_governing_datatable(&db, &w_id, &datatable_name).await?; ensure_reaches_governing_datatable(&db, &w_id, &datatable_name, &governing, &authed).await?; - ensure_instance(&governing)?; + let cluster = ensure_managed(&governing)?; let editable = ensure_governs_datatable(&db, &authed, &w_id, &governing) .await .is_ok(); let roles = if editable { - role_names(&read_role_catalog(&db).await?) + role_names(&read_role_catalog(&db, cluster).await?) } else { vec![] }; @@ -1376,7 +1388,7 @@ async fn authorize_acl_change( ) -> Result { let governing = resolve_governing_datatable(db, w_id, datatable_name).await?; ensure_governs_datatable(db, authed, w_id, &governing).await?; - ensure_instance(&governing)?; + ensure_managed(&governing)?; Ok(governing) } @@ -1386,8 +1398,14 @@ static APPLY_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1) /// provisioned before data table roles gave `custom_instance_user` none. Adds that option to its /// database and `public` privileges, and nothing else: default privileges are left alone, since a /// schema's change of owner is planned against them. Best-effort, as a grant it fails to enable is -/// refused when it runs. -async fn ensure_grant_options(client: &tokio_postgres::Client, db: &DB, dbname: &str) { +/// refused when it runs. An external instance database was created with the options, so one +/// missing there is someone's deliberate revoke and is left alone. +async fn ensure_grant_options( + client: &tokio_postgres::Client, + db: &DB, + cluster: DatatableRoleCluster, + dbname: &str, +) { let held = client .query_one( "SELECT has_database_privilege(current_database(), 'CONNECT WITH GRANT OPTION') @@ -1399,7 +1417,7 @@ async fn ensure_grant_options(client: &tokio_postgres::Client, db: &DB, dbname: ) .await .is_ok_and(|row| row.get::<_, bool>(0)); - if held { + if held || cluster != DatatableRoleCluster::Instance { return; } if let Err(e) = grant_options_as_server(db, dbname).await { @@ -1625,7 +1643,8 @@ async fn plan_datatable_acl( ) -> JsonResult { crate::datatable_acl_oss::ensure_datatable_acl_available()?; let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; - let catalog = read_role_catalog(&db).await?; + let cluster = ensure_managed(&governing)?; + let catalog = read_role_catalog(&db, cluster).await?; let (client, _notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; Ok(Json( build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?, @@ -1649,6 +1668,7 @@ async fn apply_datatable_acl( // connection could wait forever on a pool that concurrent applies, queued on the same locks, // have exhausted. let governing = authorize_acl_change(&db, &authed, &w_id, &datatable_name).await?; + let cluster = ensure_managed(&governing)?; // Applies queue on an instance-wide lock while each holds a direct connection to the instance's // Postgres; unbounded, the queue alone could exhaust its connection limit. One at a time per // server, and the ones waiting hold no connection at all. @@ -1657,7 +1677,7 @@ async fn apply_datatable_acl( .await .map_err(|e| Error::internal_err(format!("ACL apply slot closed: {e}")))?; let (mut client, mut notices, dbname) = connect_as_admin_unchecked(&db, &governing).await?; - ensure_grant_options(&client, &db, &dbname).await; + ensure_grant_options(&client, &db, cluster, &dbname).await; // Held until the change is committed: a role renamed or dropped meanwhile would change what // the plan names, and a settings save could move the entry onto another database. Taken in the @@ -1672,7 +1692,7 @@ async fn apply_datatable_acl( .fetch_optional(&mut *tx) .await? .flatten(); - let catalog = read_role_catalog_tx(&mut tx).await?; + let catalog = read_role_catalog_tx(&mut tx, cluster).await?; let plan = build_plan(&client, &dbname, &catalog, &req.target, &req.change).await?; if !entry_unchanged(&governing, entry_now) || &plan.statements != confirmed { diff --git a/backend/windmill-api-workspaces/src/datatable_clone.rs b/backend/windmill-api-workspaces/src/datatable_clone.rs index a57a2216e5..6083b7b619 100644 --- a/backend/windmill-api-workspaces/src/datatable_clone.rs +++ b/backend/windmill-api-workspaces/src/datatable_clone.rs @@ -11,8 +11,8 @@ //! The fork request makes its copies before it writes the fork: each is created, filled and — for a //! data table under roles — given the source's owners and grants. What each copy was made from //! stays in the request ([`MadeCopy`]), so the fork's transaction checks it against the source as it -//! is then, and a fork that fails drops the instance copies it made ([`drop_copies_after`]) and names -//! the others, which live on servers of the workspace's own. +//! is then, and a fork that fails drops the copies it made on a cluster Windmill manages +//! ([`drop_copies_after`]) and names the others, which live on servers of the workspace's own. use std::collections::BTreeSet; @@ -22,8 +22,8 @@ use windmill_common::error::{pg_error_message, Error, Result}; use windmill_common::utils::require_admin; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::workspaces::{ - get_datatable_resource_from_db_unchecked, DataTableDatabase, DataTableForkBehavior, - GoverningDatatable, + get_datatable_resource_from_db_unchecked, DataTableCatalogResourceType, DataTableDatabase, + DataTableForkBehavior, GoverningDatatable, }; use windmill_common::{PgDatabase, DB}; @@ -312,27 +312,38 @@ pub(crate) async fn drop_copies_after(db: &DB, copies: Vec, error: Err } async fn drop_copy(db: &DB, source_database: &DataTableDatabase, dbname: &str) -> Result<()> { - if source_database.resource_type - == windmill_common::workspaces::DataTableCatalogResourceType::Instance - { - match windmill_common::drop_unused_instance_database(db, dbname).await? { - windmill_common::Cleanup::Dropped => Ok(()), - windmill_common::Cleanup::InUse(users) => Err(Error::BadRequest(format!( - "kept, since workspaces {} now use it", - users.join(", ") - ))), - windmill_common::Cleanup::Waiter(pid) => Err(Error::BadRequest(format!( - "kept, since a request (pid {pid}) is still waiting to name it" - ))), + match source_database.resource_type { + DataTableCatalogResourceType::Instance => { + match windmill_common::drop_unused_instance_database(db, dbname).await? { + windmill_common::Cleanup::Dropped => Ok(()), + windmill_common::Cleanup::InUse(users) => Err(Error::BadRequest(format!( + "kept, since workspaces {} now use it", + users.join(", ") + ))), + windmill_common::Cleanup::Waiter(pid) => Err(Error::BadRequest(format!( + "kept, since a request (pid {pid}) is still waiting to name it" + ))), + } } - } else { + // Refused, and the copy kept, when anything names it by then. + DataTableCatalogResourceType::ExternalInstance => { + let mut tx = db.begin().await?; + windmill_common::external_instance_pg::drop_external_instance_database_unchecked( + &mut tx, dbname, None, + ) + .await?; + tx.commit().await?; + Ok(()) + } + DataTableCatalogResourceType::Postgresql => { // On a server of the workspace's own, where a resource edited meanwhile can already name it // and nothing locks such an edit: dropping it could take someone's data. - Err(Error::BadRequest( - "kept on its PostgreSQL server, where it may already be in use; drop it there once it \ - is not, before forking the same data table under this id again" - .to_string(), - )) + Err(Error::BadRequest( + "kept on its PostgreSQL server, where it may already be in use; drop it there once \ + it is not, before forking the same data table under this id again" + .to_string(), + )) + } } } @@ -349,10 +360,10 @@ async fn make_copy( request.name )) })?; - let is_instance = governing.is_instance(); + let managed = source_database.resource_type.is_windmill_managed(); // Resolved from the snapshot, not read again: the connection the copy is made on is then // exactly what these rows describe, which the fork's own clone of them is checked against. - let (server, connection): (PgDatabase, Option) = if is_instance { + let (server, connection): (PgDatabase, Option) = if managed { let server = serde_json::from_value( get_datatable_resource_from_db_unchecked(db, parent_w_id, request.name).await?, ) @@ -383,22 +394,44 @@ async fn make_copy( // Dumped before the database exists, so a source that cannot be read leaves nothing behind. // Ownership never carries over: the restore runs as the target's connection user. Grants do, - // except on the instance, where the replay below is what reproduces them. + // except on a cluster Windmill manages, which plants its own and where the replay below is what + // reproduces the roles'. let dump = pg_dump_database( &server, PgDumpOptions { schema_only: request.behavior == DataTableForkBehavior::SchemaOnly, no_owner: true, - no_acl: is_instance, + no_acl: managed, ..Default::default() }, ) .await?; - if is_instance { - windmill_common::create_custom_instance_database(db, request.dbname, "datatable").await?; - } else { - create_database_on_server(db, &server, request.dbname).await?; + match source_database.resource_type { + DataTableCatalogResourceType::Instance => { + windmill_common::create_custom_instance_database( + db, + request.dbname, + "datatable", + Some(parent_w_id), + ) + .await? + } + DataTableCatalogResourceType::ExternalInstance => { + let mut tx = db.begin().await?; + windmill_common::external_instance_pg::create_external_instance_database_unchecked( + db, + &mut tx, + request.dbname, + "datatable", + Some(parent_w_id), + ) + .await?; + tx.commit().await?; + } + DataTableCatalogResourceType::Postgresql => { + create_database_on_server(db, &server, request.dbname).await? + } } let target = PgDatabase { dbname: request.dbname.to_string(), ..server.clone() }; @@ -481,11 +514,16 @@ async fn fill( let replayed = now.datatable.permissions.is_some(); if replayed { crate::datatable_replay_oss::ensure_replay()?; - let catalog = read_role_catalog_tx(&mut tx).await?; + // The copy is on the source's cluster, so its roles come from that cluster's catalog. + let cluster = governing + .role_cluster() + .ok_or_else(|| Error::internal_err("a replayed clone is on a managed database"))?; + let catalog = read_role_catalog_tx(&mut tx, cluster).await?; // The replay leaves `CONNECT` to the catalog, and creating the database only tried to set // it: a copy `PUBLIC` could still connect to would admit logins the source turns away. windmill_common::datatable_roles::converge_connect_grants_with( db, + cluster, &target.dbname, &catalog, ) diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index df1fac203b..f762fda175 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -11,7 +11,7 @@ //! to keep that file focused on core workspace configuration. use crate::workspaces::{ - is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison, + managed_datatable_kind, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison, PgDumpOptions, }; @@ -1669,7 +1669,9 @@ async fn generate_initial_datatable_migration( // without what a replay elsewhere cannot run: the replaying user owns none of this // database's objects, and the grants Windmill plants in an instance database (`ALTER // DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server. - let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?; + let no_acl = managed_datatable_kind(&db, &w_id, &datatable_name) + .await? + .is_some(); let dump_file = pg_dump_database( &pg_db, PgDumpOptions { diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 185fe17b25..458e63a973 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2300,7 +2300,8 @@ struct DataTableTables { schemas: TableListMap, #[serde(skip_serializing_if = "Option::is_none")] error: Option, - /// On the instance database: the only kind that can be under roles or have its access edited. + /// On a database Windmill manages, on its own cluster or the external one: the only kinds that + /// can be under roles or have their access edited. instance: bool, permissioned: bool, /// The roles this caller may connect as, by name; empty when not under roles. @@ -2557,7 +2558,7 @@ async fn list_one_datatable_tables( }; let result: Result<()> = async { let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?; - entry.instance = governing.is_instance(); + entry.instance = governing.role_cluster().is_some(); let usable = crate::datatable_permissions_oss::usable_datatable_roles(db, authed, w_id, &governing) .await?; @@ -3082,6 +3083,25 @@ pub(crate) async fn resolve_pg_source_checked( w_id: &str, source: &str, ) -> Result { + Ok( + resolve_pg_source_checked_with_kind(db, user_db, authed, w_id, source) + .await? + .0, + ) +} + +/// [`resolve_pg_source_checked`], also reporting the kind of Windmill-managed database behind the +/// source, `None` for a user resource. Callers that guard a managed connection MUST take the kind +/// from here rather than ask separately: between two reads a save can flip the entry, leaving the guards of one kind +/// applied to the connection of the other. +pub(crate) async fn resolve_pg_source_checked_with_kind( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + source: &str, +) -> Result<(PgDatabase, Option)> { + let mut managed_kind = None; let db_resource = if let Some(name) = source.strip_prefix("datatable://") { windmill_common::workspaces::ensure_datatable_admin_access( db, @@ -3090,7 +3110,13 @@ pub(crate) async fn resolve_pg_source_checked( &DatatableAccess::Authed(authed.to_authed_ref()), ) .await?; - get_datatable_resource_from_db_unchecked(db, w_id, name).await? + let (connection, kind) = + windmill_common::workspaces::get_datatable_connection_and_kind_unchecked( + db, w_id, name, + ) + .await?; + managed_kind = kind; + connection } else if let Some(path) = source.strip_prefix("$res:") { let db_with_authed = windmill_common::db::DbWithOptAuthed::from_authed( authed, @@ -3123,27 +3149,37 @@ pub(crate) async fn resolve_pg_source_checked( ))); }; - serde_json::from_value(db_resource) - .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) + let pg: PgDatabase = serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))?; + Ok((pg, managed_kind)) } -/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL -/// rather than a user resource. -pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result { +/// The kind of the database backing the data table `name` when Windmill manages it (on its own +/// cluster or the external one), `None` when it is a user resource. +pub(crate) async fn managed_datatable_kind( + db: &DB, + w_id: &str, + name: &str, +) -> Result> { // Resolved rather than read: a pointer entry owns no database of its own, so only the entry it - // lands on can answer. A name that resolves to nothing keeps the historical `false`. + // lands on can answer. A name that resolves to nothing keeps the historical `None`. Ok(resolve_governing_datatable(db, w_id, name) .await .ok() .and_then(|g| g.datatable.database) - .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)) + .map(|d| d.resource_type) + .filter(|kind| kind.is_windmill_managed())) } /// Same, for the `datatable://` / `$res:` form the import endpoints take. -async fn is_instance_datatable_source(db: &DB, w_id: &str, source: &str) -> Result { +async fn managed_datatable_source_kind( + db: &DB, + w_id: &str, + source: &str, +) -> Result> { match source.strip_prefix("datatable://") { - Some(name) => is_instance_datatable(db, w_id, name).await, - None => Ok(false), + Some(name) => managed_datatable_kind(db, w_id, name).await, + None => Ok(None), } } @@ -3242,13 +3278,7 @@ pub(crate) async fn pg_dump_database( if let Some(ref password) = pg_db.password { cmd.env("PGPASSWORD", password); } - - if let Some(ref sslmode) = pg_db.sslmode { - cmd.env("PGSSLMODE", sslmode); - } - if let Some(options) = pg_db.non_empty_options() { - cmd.env("PGOPTIONS", options); - } + let _root_cert = apply_pg_tls_env(&mut cmd, pg_db)?; let output = cmd .output() @@ -3366,7 +3396,7 @@ async fn comment_out_unsupported_settings( /// A psql invocation against `pg_db`, carrying the connection settings the CLI reads /// from the environment. -fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { +fn psql_command(pg_db: &PgDatabase) -> Result<(tokio::process::Command, Option)> { let mut cmd = tokio::process::Command::new("psql"); cmd.arg("--host") .arg(&pg_db.host) @@ -3383,13 +3413,92 @@ fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { if let Some(ref password) = pg_db.password { cmd.env("PGPASSWORD", password); } + let root_cert = apply_pg_tls_env(&mut cmd, pg_db)?; + Ok((cmd, root_cert)) +} + +/// Give libpq the TLS settings `PgDatabase::connect` applies. The returned file holds the root +/// certificate `PGSSLROOTCERT` names, so it must outlive the command. +fn apply_pg_tls_env( + cmd: &mut tokio::process::Command, + pg_db: &PgDatabase, +) -> Result> { if let Some(ref sslmode) = pg_db.sslmode { cmd.env("PGSSLMODE", sslmode); } if let Some(options) = pg_db.non_empty_options() { cmd.env("PGOPTIONS", options); } - cmd + if let Some(pem) = pg_db + .root_certificate_pem + .as_deref() + .filter(|p| !p.is_empty()) + { + let file = DumpFile::new()?; + std::fs::write(&file.path, pem) + .map_err(|e| Error::internal_err(format!("Failed to write root certificate: {e}")))?; + cmd.env("PGSSLROOTCERT", &file.path); + return Ok(Some(file)); + } + // Only a connection that asked to be verified against the system trust store. Without a file, + // libpq's own default would look for `~/.postgresql/root.crt` and refuse a verify-* mode. libpq + // takes the special `system` value with verify-full only, so verify-ca needs the bundle itself. + if pg_db.accept_invalid_certs == Some(false) { + match pg_db.sslmode.as_deref() { + Some("verify-full") => { + cmd.env("PGSSLROOTCERT", "system"); + } + Some("verify-ca") => { + if let Some(bundle) = windmill_common::system_ca_bundle() { + cmd.env("PGSSLROOTCERT", bundle); + } + } + _ => {} + } + } + Ok(None) +} + + +#[cfg(test)] +mod pg_tls_env_tests { + use super::apply_pg_tls_env; + use windmill_common::PgDatabase; + + fn root_cert_env(sslmode: &str) -> Option { + let pg_db = PgDatabase { + host: "db".to_string(), + user: None, + password: None, + port: None, + sslmode: Some(sslmode.to_string()), + dbname: "d".to_string(), + root_certificate_pem: None, + accept_invalid_certs: Some(false), + use_iam_auth: None, + region: None, + options: None, + }; + let mut cmd = tokio::process::Command::new("psql"); + apply_pg_tls_env(&mut cmd, &pg_db).unwrap(); + cmd.as_std() + .get_envs() + .find(|(k, _)| *k == "PGSSLROOTCERT") + .and_then(|(_, v)| v.map(|v| v.to_os_string())) + } + + #[test] + fn system_roots_only_through_verify_full() { + assert_eq!( + root_cert_env("verify-full").as_deref(), + Some("system".as_ref()) + ); + // libpq refuses `sslrootcert=system` with verify-ca, which would fail every dump and restore. + assert_ne!( + root_cert_env("verify-ca").as_deref(), + Some("system".as_ref()) + ); + } } /// GUC names the server backing `pg_db` knows about. @@ -3399,7 +3508,8 @@ fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command { /// and an unset mode, where `PgDatabase::connect` would hand a TLS-only server a /// plaintext socket and fail before the import ever starts. async fn server_setting_names(pg_db: &PgDatabase) -> Result> { - let output = psql_command(pg_db) + let (mut cmd, _root_cert) = psql_command(pg_db)?; + let output = cmd .arg("--tuples-only") .arg("--no-align") .arg("--command") @@ -3436,7 +3546,8 @@ pub(crate) async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) let supported_settings = server_setting_names(target_db).await?; comment_out_unsupported_settings(dump_file, &supported_settings).await?; - let output = psql_command(target_db) + let (mut cmd, _root_cert) = psql_command(target_db)?; + let output = cmd .arg("--set") .arg("ON_ERROR_STOP=1") .arg("--single-transaction") @@ -3494,9 +3605,39 @@ async fn create_pg_database( } } - if is_instance_datatable_source(&db, &w_id, &req.source).await? { - windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") + if let Some(source_kind) = managed_datatable_source_kind(&db, &w_id, &req.source).await? { + // Held until the copy is registered, as a rename migrates reservations to the new id under + // it once the old one is archived: a copy registered after that would be reserved for an + // id nothing answers on. + let mut tx = db.begin().await?; + windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?; + let live = sqlx::query_scalar::<_, bool>("SELECT NOT deleted FROM workspace WHERE id = $1") + .bind(&w_id) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + if !live { + return Err(Error::BadRequest(format!("Workspace '{w_id}' is archived"))); + } + if source_kind == DataTableCatalogResourceType::ExternalInstance { + windmill_common::external_instance_pg::create_external_instance_database_unchecked( + &db, + &mut tx, + &req.target_dbname, + "datatable", + Some(&w_id), + ) .await?; + } else { + windmill_common::create_custom_instance_database( + &db, + &req.target_dbname, + "datatable", + Some(&w_id), + ) + .await?; + } + tx.commit().await?; } else { let source_pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; @@ -3583,12 +3724,12 @@ pub(crate) async fn ensure_datatable_is_clonable( let governing = resolve_governing_datatable(db, w_id, name).await?; // The copy has to name a database of its own. A resource-backed entry reached through a // pointer names one this workspace does not own, so there is nothing here to repoint. - let is_instance = governing + let is_managed = governing .datatable .database .as_ref() - .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance); - if governing.workspace_id != w_id && !is_instance { + .is_some_and(|d| d.resource_type.is_windmill_managed()); + if governing.workspace_id != w_id && !is_managed { return Err(Error::BadRequest(format!( "Data table '{name}' points at a resource-backed data table in another workspace \ and cannot be copied; fork it from the workspace that owns it." @@ -3642,9 +3783,11 @@ async fn import_pg_database( } let schema_only = req.fork_behavior == DataTableForkBehavior::SchemaOnly; - let source_pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; - let mut target_pg = - resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.target).await?; + let mut fork_lock: Option> = None; + let (source_pg, source_kind) = + resolve_pg_source_checked_with_kind(&db, &user_db, &authed, &w_id, &req.source).await?; + let (mut target_pg, target_kind) = + resolve_pg_source_checked_with_kind(&db, &user_db, &authed, &w_id, &req.target).await?; if let Some(ref override_dbname) = req.target_dbname_override { if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? { @@ -3654,6 +3797,29 @@ async fn import_pg_database( .to_string(), )); } + // The kind the connection above was built from, never a second read: an entry flipped + // to a resource between the two would keep the managed connection here and lose the + // check that decides which copy it may fill. + if let Some(kind) = target_kind { + // Held until the restore is done: fork finalization takes the first, and every + // save newly naming a database, in any workspace, the second. Nothing may start + // using this database while `psql` is still filling it. + let mut tx = db.begin().await?; + windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?; + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut tx, + [override_dbname.as_str()], + ) + .await?; + windmill_common::ensure_fork_database_available_to( + &mut tx, + kind, + override_dbname, + &w_id, + ) + .await?; + fork_lock = Some(tx); + } } target_pg.dbname = override_dbname.clone(); } @@ -3663,8 +3829,7 @@ async fn import_pg_database( // what it creates it owns. Grants do, except around an instance data table — Windmill // plants `custom_instance_user` grants in one, which nothing else can replay. Elsewhere // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access. - let no_acl = is_instance_datatable_source(&db, &w_id, &req.target).await? - || is_instance_datatable_source(&db, &w_id, &req.source).await?; + let no_acl = target_kind.is_some() || source_kind.is_some(); let dump_file = pg_dump_database( &source_pg, @@ -3672,6 +3837,9 @@ async fn import_pg_database( ) .await?; pg_import_dump(&target_pg, &dump_file).await?; + if let Some(tx) = fork_lock { + tx.commit().await?; + } Ok(format!( "Imported from '{}' into '{}'", @@ -3772,39 +3940,76 @@ async fn edit_ducklake_config( ) .await?; - let old_ducklakes = sqlx::query_scalar!( - r#" - SELECT ws.ducklake->'ducklakes' AS ducklake_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id + // Under the row lock the save writes with, taken before the database locks below as fork + // cleanup takes the two. + let old_ducklakes = sqlx::query_scalar::<_, Option>( + "SELECT ws.ducklake->'ducklakes' FROM workspace_settings ws + WHERE ws.workspace_id = $1 FOR UPDATE", ) + .bind(&w_id) .fetch_one(&mut *tx) .await? .unwrap_or(serde_json::Value::Null); let old_ducklakes: HashMap = serde_json::from_value(old_ducklakes).unwrap_or_default(); - // Check that non-superadmins are not abusing Instance databases - if !is_superadmin { - for (name, dl) in new_config.settings.ducklakes.iter() { - if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { - let old_dl = old_ducklakes.get(name); - if old_dl.is_none() - || old_dl.unwrap().catalog.resource_type - != DucklakeCatalogResourceType::Instance - || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path - { - return Err(Error::BadRequest( - "Only superadmins can create or modify ducklakes with Instance databases" - .to_string(), - )); - } - } + // Check that non-superadmins are not abusing Instance databases. An unchanged catalog is left + // alone either way, so a downgraded instance can still save lakes that already name an + // external instance database. + for (name, dl) in new_config.settings.ducklakes.iter() { + let kind = &dl.catalog.resource_type; + if !matches!( + kind, + DucklakeCatalogResourceType::Instance | DucklakeCatalogResourceType::ExternalInstance + ) { + continue; + } + let unchanged = old_ducklakes.get(name).is_some_and(|old| { + &old.catalog.resource_type == kind + && old.catalog.resource_path == dl.catalog.resource_path + }); + if unchanged { + continue; + } + // Before the registration check, whose refusal would otherwise tell a workspace admin + // which databases exist on the cluster. + if !is_superadmin { + return Err(Error::BadRequest( + "Only superadmins can create or modify ducklakes with Instance databases" + .to_string(), + )); + } + if *kind == DucklakeCatalogResourceType::ExternalInstance { + windmill_common::external_instance_pg::ensure_external_instance_available()?; + windmill_common::external_instance_pg::ensure_external_instance_database_registered( + &mut tx, + &dl.catalog.resource_path, + ) + .await?; + } else { + windmill_common::workspaces::ensure_instance_pg_available(&mut *tx).await?; } } + // Fork cleanup decides nothing uses an instance database under this lock, so a catalog newly + // put on one must not commit between its check and its drop. + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut *tx, + new_config + .settings + .ducklakes + .iter() + .filter(|(name, dl)| { + dl.catalog.resource_type == DucklakeCatalogResourceType::Instance + && old_ducklakes.get(name.as_str()).is_none_or(|old| { + old.catalog.resource_type != DucklakeCatalogResourceType::Instance + || old.catalog.resource_path != dl.catalog.resource_path + }) + }) + .map(|(_, dl)| dl.catalog.resource_path.as_str()), + ) + .await?; + let config: serde_json::Value = serde_json::to_value(&new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; @@ -3860,6 +4065,8 @@ async fn edit_datatable_config( let is_superadmin = require_super_admin(&db, &authed).await.is_ok(); let mut tx = db.begin().await?; + // Ahead of the settings row, as fork cleanup of this workspace takes the two. + windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?; windmill_common::lock_instance_databases( &mut tx, new_config.settings.datatables.values().filter_map(|dt| { @@ -3983,6 +4190,7 @@ async fn edit_datatable_config( // so these line up with the `datatable_configured` adoption counts. created_substrates.push(match dt.database.as_ref().map(|d| d.resource_type) { Some(DataTableCatalogResourceType::Instance) => "instance", + Some(DataTableCatalogResourceType::ExternalInstance) => "external_instance", Some(DataTableCatalogResourceType::Postgresql) => "postgresql", None => "reference", }); @@ -4018,18 +4226,20 @@ async fn edit_datatable_config( ))); } // Carrying the block onto a resource-backed entry would produce a data table the chokepoint - // refuses on every job — a save that succeeds and breaks everything afterwards. Refuse it - // instead: turning roles off first is one step, and it keeps discarding an access decision - // something somebody chose rather than a side effect of moving a database. + // refuses on every job — a save that succeeds and breaks everything afterwards — and onto + // the other managed cluster, one whose role ids name nothing in that cluster's catalog. + // Refuse it instead: turning roles off first is one step, and it keeps discarding an access + // decision something somebody chose rather than a side effect of moving a database. + let old_kind = old.and_then(|old| old.database.as_ref()).map(|d| d.resource_type); if dt.permissions.is_some() && dt .database .as_ref() - .is_some_and(|d| d.resource_type != DataTableCatalogResourceType::Instance) + .is_some_and(|d| Some(d.resource_type) != old_kind) { return Err(Error::BadRequest(format!( - "Data table '{name}' is under roles, which only a data table on the instance \ - database can be. Turn its roles off before moving it to a PostgreSQL resource." + "Data table '{name}' is under roles, which belong to the cluster its database is \ + on. Turn its roles off before moving it to another kind of database." ))); } // A pointer names no database of its own, so the form's empty `database` is correct there. @@ -4054,35 +4264,52 @@ async fn edit_datatable_config( // Check that non-superadmins are not abusing Instance databases, which reach a database this // workspace does not own. Pointing an entry at another workspace's data table is not checked // here because it cannot be requested at all: `reference` is overwritten from the stored entry - // above, for every caller. + // above, for every caller. An unchanged entry is left alone either way, so a downgraded + // instance can still save settings that already name an external instance database. // // Compared against the entry the carried fields came from, not the one stored under the same // name: otherwise swapping two names keeps each database in place while moving a clone's // `governed_by` off the copy it governs. - if !is_superadmin { - for (name, dt) in new_config.settings.datatables.iter() { - let old_dt = old_datatables.get( + for (name, dt) in new_config.settings.datatables.iter() { + let Some(database) = dt + .database + .as_ref() + .filter(|d| d.resource_type.is_windmill_managed()) + else { + continue; + }; + let unchanged = old_datatables + .get( rename_src .get(name.as_str()) .copied() .unwrap_or(name.as_str()), - ); - if dt - .database - .as_ref() - .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) - { - let unchanged = old_dt.and_then(|o| o.database.as_ref()).is_some_and(|o| { - o.resource_type == DataTableCatalogResourceType::Instance - && Some(&o.resource_path) == dt.database.as_ref().map(|d| &d.resource_path) - }); - if !unchanged { - return Err(Error::BadRequest( - "Only superadmins can create or modify data tables with Instance databases" - .to_string(), - )); - } - } + ) + .and_then(|o| o.database.as_ref()) + .is_some_and(|o| { + o.resource_type == database.resource_type + && o.resource_path == database.resource_path + }); + if unchanged { + continue; + } + // Before the registration check, whose refusal would otherwise tell a workspace admin + // which databases exist on the cluster. + if !is_superadmin { + return Err(Error::BadRequest( + "Only superadmins can create or modify data tables with Instance databases" + .to_string(), + )); + } + if database.resource_type == DataTableCatalogResourceType::ExternalInstance { + windmill_common::external_instance_pg::ensure_external_instance_available()?; + windmill_common::external_instance_pg::ensure_external_instance_database_registered( + &mut tx, + &database.resource_path, + ) + .await?; + } else { + windmill_common::workspaces::ensure_instance_pg_available(&mut *tx).await?; } } @@ -4102,7 +4329,7 @@ async fn edit_datatable_config( // entry through a declared rename alone, and a settings sync never declares one, so an entry // without roles that newly points at such a database — a name added, or an existing one // repointed — would answer everyone there as `admin`. That holds whichever workspace governs it. - let newly_pointed: Vec<(&String, &str)> = new_config + let newly_pointed: Vec<(&String, DataTableCatalogResourceType, &str)> = new_config .settings .datatables .iter() @@ -4112,7 +4339,7 @@ async fn edit_datatable_config( let db = dt .database .as_ref() - .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?; + .filter(|d| d.resource_type.is_windmill_managed())?; let lookup = rename_src .get(name.as_str()) .copied() @@ -4124,38 +4351,72 @@ async fn edit_datatable_config( old_db.resource_type != db.resource_type || old_db.resource_path != db.resource_path }); - repointed.then_some((name, db.resource_path.as_str())) + repointed.then_some((name, db.resource_type, db.resource_path.as_str())) }) .collect(); // Another workspace turning roles on for the same database holds only its own settings row, so - // without this the scan below could read past its uncommitted write. + // without this the scan below could read past its uncommitted write. Every managed database + // this save newly names is locked, not just the ones the scan is about: fork cleanup takes the + // same lock to decide nothing uses the database it is dropping. + let newly_named: std::collections::BTreeSet<&str> = new_config + .settings + .datatables + .iter() + .filter_map(|(name, dt)| { + let db = dt + .database + .as_ref() + .filter(|d| d.resource_type == DataTableCatalogResourceType::Instance)?; + let lookup = rename_src + .get(name.as_str()) + .copied() + .unwrap_or(name.as_str()); + old_datatables + .get(lookup) + .and_then(|old| old.database.as_ref()) + .is_none_or(|old_db| { + old_db.resource_type != db.resource_type + || old_db.resource_path != db.resource_path + }) + .then_some(db.resource_path.as_str()) + }) + .collect(); windmill_common::datatable_roles::lock_instance_databases_governance( &mut *tx, - newly_pointed.iter().map(|(_, dbname)| *dbname), + newly_pointed + .iter() + .map(|(_, _, dbname)| *dbname) + .chain(newly_named.iter().copied()), ) .await?; - let governed_elsewhere: Vec = if newly_pointed.is_empty() { + let governed_elsewhere: Vec<(String, String)> = if newly_pointed.is_empty() { vec![] } else { - sqlx::query_scalar( - "SELECT DISTINCT dt.value->'database'->>'resource_path' FROM workspace_settings ws + sqlx::query_as( + "SELECT DISTINCT dt.value->'database'->>'resource_type', + dt.value->'database'->>'resource_path' + FROM workspace_settings ws CROSS JOIN LATERAL jsonb_each(COALESCE(ws.datatable->'datatables', '{}'::jsonb)) dt WHERE ws.workspace_id <> $1 AND (dt.value ? 'permissions' OR dt.value ? 'governed_by') - AND dt.value->'database'->>'resource_type' = 'instance'", + AND dt.value->'database'->>'resource_type' IN ('instance', 'external_instance')", ) .bind(&w_id) .fetch_all(&mut *tx) .await? }; - for (name, dbname) in newly_pointed { + for (name, kind, dbname) in newly_pointed { let governed_here = old_datatables.values().any(|old| { (old.permissions.is_some() || old.governed_by.is_some()) - && old.database.as_ref().is_some_and(|d| { - d.resource_type == DataTableCatalogResourceType::Instance - && d.resource_path == dbname - }) + && old + .database + .as_ref() + .is_some_and(|d| d.resource_type == kind && d.resource_path == dbname) }); - if governed_here || governed_elsewhere.iter().any(|g| g == dbname) { + if governed_here + || governed_elsewhere + .iter() + .any(|(k, p)| k == kind.as_ref() && p == dbname) + { return Err(Error::BadRequest(format!( "Data table '{name}' would point at database '{dbname}', which a data table under \ roles uses, without carrying those roles: everyone reaching '{name}' would connect \ @@ -8429,13 +8690,14 @@ async fn point_kept_datatables_at_parent( if dt.reference.is_some() { continue; } - // Only instance databases. A resource-backed data table names a resource, and the settings - // clone gave the fork its own copy of that resource in its own workspace — pointing at the - // parent's entry would silently move the fork onto the parent's resource instead. + // Only instance databases, on either cluster. A resource-backed data table names a + // resource, and the settings clone gave the fork its own copy of that resource in its own + // workspace — pointing at the parent's entry would silently move the fork onto the + // parent's resource instead. if dt .database .as_ref() - .is_none_or(|d| d.resource_type != DataTableCatalogResourceType::Instance) + .is_none_or(|d| !d.resource_type.is_windmill_managed()) { continue; } @@ -8609,11 +8871,39 @@ async fn apply_forked_datatable( })?, }; - if database.resource_type == DataTableCatalogResourceType::Instance { + if database.resource_type == DataTableCatalogResourceType::ExternalInstance { + windmill_common::external_instance_pg::ensure_external_instance_database_registered( + tx, + &fdt.new_dbname, + ) + .await?; + } + if database.resource_type.is_windmill_managed() { + // Held until the fork commits, as every save newly naming a database takes it: none may + // claim the copy between the check below and this fork's entry landing on it. + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut **tx, + [fdt.new_dbname.as_str()], + ) + .await?; + } + if database.resource_type.is_windmill_managed() + && !windmill_api_auth::is_super_admin_authed(db, authed).await? + { + windmill_common::ensure_fork_database_available_to( + &mut **tx, + database.resource_type, + &fdt.new_dbname, + parent_w_id, + ) + .await?; + } + if database.resource_type.is_windmill_managed() { // The whole `database` object, not just its `resource_path`: a pointer entry has none to - // patch. `reference` goes with it — exactly one of the two may be set. + // patch. `reference` goes with it — exactly one of the two may be set. The copy was created + // on the same cluster as its source, so it keeps the source's kind. let new_database = serde_json::json!({ - "resource_type": "instance", + "resource_type": database.resource_type, "resource_path": &fdt.new_dbname, }); sqlx::query( @@ -9184,6 +9474,11 @@ async fn write_workspace_fork( .bind(&nw.id) .execute(&mut *tx) .await?; + // Before the settings clone reads the parent's data tables: a pointer this fork ends up with + // must not be written after cleanup of the parent decided that nothing points at its copies. + // Also before the external cluster's lifecycle lock, which finalizing an external copy takes: + // fork cleanup takes the two in this order. + windmill_common::workspaces::lock_fork_datatables(&mut tx, &parent_workspace_id).await?; if nw.is_dev_workspace { // The checks above ran outside a transaction, so the parent's eligibility and the chain's @@ -9328,6 +9623,32 @@ async fn write_workspace_fork( ))); } } + // Saves name an external database under the external cluster's lifecycle lock instead. + let external_copies: Vec<&str> = copies + .iter() + .filter(|c| { + c.source_database.resource_type == DataTableCatalogResourceType::ExternalInstance + }) + .map(|c| c.dbname.as_str()) + .collect(); + if !external_copies.is_empty() { + windmill_common::external_instance_pg::lock_external_instance_pg_state(&mut tx).await?; + } + for dbname in &external_copies { + let uses = windmill_common::workspaces::managed_database_uses( + &mut tx, + DataTableCatalogResourceType::ExternalInstance, + dbname, + None, + ) + .await?; + if !uses.is_empty() { + return Err(Error::BadRequest(format!( + "Database '{dbname}' copied for this fork is already used by {}; fork again", + uses.join(", ") + ))); + } + } // Update forked datatable settings to point to new databases for fdt in &nw.forked_datatables { diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 9039bda2ef..8ca98f66d1 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -56,6 +56,11 @@ pub(crate) async fn change_workspace_id( let mut tx = db.begin().await?; + // The settings copy below carries every data table entry to the new id, which fork cleanup of + // the old id cannot see until this commits: without the lock it could drop a copy the renamed + // workspace goes on using. Before the pairing lock, as forking takes the two in that order. + windmill_common::workspaces::lock_fork_datatables(&mut tx, &old_id).await?; + // A rename rewrites the workspace's dev flag and reparents its children, so it decides on the // same state the pairing handlers do: without this lock a concurrent create/attach could commit // an active dev workspace under the shell this rename is about to archive. Both ids, since the @@ -869,6 +874,10 @@ pub(crate) async fn change_workspace_id( } } + // After every workspace_settings write above: fork cleanup locks a settings row before the + // registry, so taking the registry first here would deadlock with it. + migrate_fork_reservations(&mut tx, &old_id, &rw.new_id).await?; + // Audit log in the same transaction as the workspace changes audit_log( &mut *tx, @@ -937,6 +946,14 @@ pub(crate) async fn change_workspace_id( let (_schedules_count, canceled_count, _deleted_tokens_count) = archive_workspace_impl(&db, &old_id, &authed.username, None).await?; + // The old id stays live between the commit above and the archive, and a fork copy created for + // it in that window registers under it. Creation checks the workspace is live under the fork + // lock, so once this has run under it, no copy can be reserved for the old id any more. + let mut tx = db.begin().await?; + windmill_common::workspaces::lock_fork_datatables(&mut tx, &old_id).await?; + migrate_fork_reservations(&mut tx, &old_id, &rw.new_id).await?; + tx.commit().await?; + info!( "Workspace id change completed: moved {} to {}, archived old workspace", old_id, rw.new_id @@ -948,6 +965,50 @@ pub(crate) async fn change_workspace_id( )) } +/// A fork copy reserved for the old id would otherwise be unreachable: its creator cannot import +/// into it or finish its fork under the new id, and nothing else would ever drop it. +async fn migrate_fork_reservations( + tx: &mut Transaction<'_, Postgres>, + old_id: &str, + new_id: &str, +) -> Result<()> { + const MIGRATE: &str = r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', ( + SELECT COALESCE(jsonb_object_agg(k, CASE WHEN v->>'workspace_id' = $1 + THEN jsonb_set(v, '{workspace_id}', to_jsonb($2::text)) ELSE v END), '{}'::jsonb) + FROM jsonb_each(COALESCE(value->'databases', '{}'::jsonb)) AS e(k, v) + )) + WHERE name = $3"#; + sqlx::query(MIGRATE) + .bind(old_id) + .bind(new_id) + .bind("custom_instance_pg_databases") + .execute(&mut **tx) + .await?; + // External copies are registered in the cluster state, which its writers rewrite whole under + // the lifecycle lock. Only taken when there is something to move, as setup can hold it for as + // long as the cluster takes to answer. + let state = windmill_common::global_settings::EXTERNAL_INSTANCE_PG_STATE_SETTING; + let reserved = sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM global_settings, jsonb_each(value->'databases') AS e(k, v) + WHERE name = $1 AND jsonb_typeof(value->'databases') = 'object' + AND v->>'workspace_id' = $2)", + ) + .bind(state) + .bind(old_id) + .fetch_one(&mut **tx) + .await?; + if reserved { + windmill_common::external_instance_pg::lock_external_instance_pg_state(tx).await?; + sqlx::query(MIGRATE) + .bind(old_id) + .bind(new_id) + .bind(state) + .execute(&mut **tx) + .await?; + } + Ok(()) +} + #[derive(Deserialize)] pub(crate) struct DeleteWorkspaceQuery { pub(crate) only_delete_forks: Option, @@ -1430,9 +1491,7 @@ pub async fn drop_forked_datatable_databases( _ => continue, }; - if database.resource_type - == windmill_common::workspaces::DataTableCatalogResourceType::Instance - { + if database.resource_type.is_windmill_managed() { let db_to_drop = &database.resource_path; if !db_to_drop.starts_with("wm_fork_") { errors.push(format!( @@ -1441,7 +1500,112 @@ pub async fn drop_forked_datatable_databases( )); continue; } - if let Err(e) = windmill_common::drop_custom_instance_database(&db, db_to_drop).await { + // The fork's own entry is what is going away; anything else still reaching the copy, + // a child fork's pointer at this entry included, keeps it. The lock keeps a child fork + // from gaining such a pointer before the drop. + // A task of its own, so a client going away cannot stop it between dropping the + // database and committing the entry's removal. + let dropped = tokio::spawn({ + let (db, w_id, dt_name, db_to_drop) = ( + db.clone(), + w_id.clone(), + dt_name.clone(), + db_to_drop.clone(), + ); + let resource_type = database.resource_type; + async move { + let mut tx = db.begin().await?; + // The three locks a settings save takes, in its order: this workspace's data + // tables, its settings row, and the database itself. Without them a save could + // rename this entry, or point another one here, either side of the check below. + windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?; + // The snapshot above was read unlocked: a save committing since could have + // repointed this entry, and the entry is removed below whatever it names by then. + let current = sqlx::query_scalar::<_, Option>( + "SELECT datatable->'datatables'->$2 FROM workspace_settings + WHERE workspace_id = $1 FOR UPDATE", + ) + .bind(&w_id) + .bind(&dt_name) + .fetch_optional(&mut *tx) + .await? + .flatten() + .and_then(|v| serde_json::from_value::(v).ok()); + if !current.is_some_and(|dt| { + dt.forked_from.is_some() + && dt.database.is_some_and(|d| { + d.resource_type == resource_type && d.resource_path == db_to_drop + }) + }) { + return Err(Error::BadRequest( + "the data table changed while it was being cleaned up".to_string(), + )); + } + let external = resource_type + == windmill_common::workspaces::DataTableCatalogResourceType::ExternalInstance; + if external { + // Before the governance lock, in the order settings saves and fork + // finalization take the two. + windmill_common::external_instance_pg::lock_external_instance_pg_state( + &mut tx, + ) + .await?; + } + windmill_common::datatable_roles::lock_instance_databases_governance( + &mut tx, + [db_to_drop.as_str()], + ) + .await?; + // Before the entry is removed: child fork pointers are found through it. + let uses = windmill_common::workspaces::managed_database_uses( + &mut tx, + resource_type, + &db_to_drop, + Some((w_id.as_str(), dt_name.as_str())), + ) + .await?; + if !uses.is_empty() { + return Err(Error::BadRequest(format!( + "it is still used by {}", + uses.join(", ") + ))); + } + // The entry goes with the database: a fork this one is cloned into afterwards + // must not inherit a pointer at a data table whose database is gone. + sqlx::query( + "UPDATE workspace_settings SET datatable = datatable #- ARRAY['datatables', $2] + WHERE workspace_id = $1", + ) + .bind(&w_id) + .bind(&dt_name) + .execute(&mut *tx) + .await?; + if external { + // Checks the uses of the database itself, and unregisters it. + windmill_common::external_instance_pg::drop_external_instance_database_unchecked( + &mut tx, + &db_to_drop, + Some((w_id.as_str(), dt_name.as_str())), + ) + .await?; + } else { + windmill_common::drop_custom_instance_database_keep_entry(&db, &db_to_drop) + .await?; + sqlx::query( + "UPDATE global_settings SET value = value #- ARRAY['databases', $1] + WHERE name = 'custom_instance_pg_databases'", + ) + .bind(&db_to_drop) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok::<_, Error>(()) + } + }) + .await + .unwrap_or_else(|e| Err(Error::internal_err(format!("cleanup task failed: {e}")))); + if let Err(e) = dropped { errors.push(format!( "Could not drop instance database '{}' for datatable://{}: {}", db_to_drop, dt_name, e @@ -1808,7 +1972,17 @@ async fn resolve_fork_catalog_pg( "ducklake://{ducklake_name}: malformed registry catalog identity `{catalog}`" )) })?; - let catalog_resource = if resource_type == "instance" { + let catalog_resource = if resource_type == "external_instance" { + serde_json::to_value( + windmill_common::external_instance_pg::external_instance_connection_unchecked( + db, + resource_path, + false, + ) + .await?, + ) + .map_err(|e| Error::internal_err(format!("serializing pg creds: {e}")))? + } else if resource_type == "instance" { let mut pg_creds = windmill_common::PgDatabase::parse_uri( &windmill_common::get_database_url().await?.as_str().await, )?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4a1ce62bad..cf373241d4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1572,6 +1572,104 @@ paths: schema: type: object + /settings/external_instance_pg/status: + get: + summary: Returns whether the external instance cluster is configured and how its last setup went + operationId: getExternalInstancePgStatus + tags: + - setting + responses: + "200": + description: external instance cluster status + content: + application/json: + schema: + $ref: "#/components/schemas/ExternalInstancePgStatus" + + /settings/external_instance_pg/setup: + post: + summary: Sets up the external instance cluster with its saved admin login, optionally rotating the passwords Windmill manages on it (enterprise edition only) + operationId: setupExternalInstancePg + tags: + - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + rotate_passwords: + type: boolean + responses: + "200": + description: the setup report, also stored as the last setup + content: + application/json: + schema: + $ref: "#/components/schemas/ExternalInstancePgSetupReport" + + /settings/external_instance_pg/databases: + get: + summary: Lists the databases Windmill created on the external instance cluster, with the workspaces whose data tables, Ducklake catalogs or pending fork cleanups use each + operationId: listExternalInstancePgDatabases + tags: + - setting + responses: + "200": + description: databases by name + content: + application/json: + schema: + type: object + additionalProperties: + $ref: "#/components/schemas/CustomInstanceDb" + + /settings/external_instance_pg/databases/{name}: + post: + summary: Creates a database on the external instance cluster (enterprise edition only) + operationId: createExternalInstancePgDatabase + tags: + - setting + parameters: + - name: name + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + tag: + $ref: "#/components/schemas/CustomInstanceDbTag" + responses: + "200": + description: database created + content: + application/json: + schema: {} + delete: + summary: Drops a database Windmill created on the external instance cluster, refused while a data table, Ducklake catalog or pending fork cleanup uses it + operationId: dropExternalInstancePgDatabase + tags: + - setting + parameters: + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: database dropped + content: + application/json: + schema: {} + /settings/list_custom_instance_pg_databases: post: summary: Returns the set-up statuses of custom instance pg databases @@ -1590,13 +1688,19 @@ paths: /settings/datatable_roles: get: - summary: list the instance's data table roles + summary: list the data table roles of one Windmill-managed Postgres cluster operationId: listInstanceDatatableRoles tags: - setting + parameters: + - in: query + name: cluster + required: false + schema: + $ref: "#/components/schemas/DatatableRoleCluster" responses: "200": - description: the instance role catalog + description: the cluster's role catalog content: application/json: schema: @@ -1604,7 +1708,7 @@ paths: items: $ref: "#/components/schemas/InstanceDatatableRole" post: - summary: create a data table role on the instance's Postgres cluster + summary: create a data table role on a Windmill-managed Postgres cluster operationId: createInstanceDatatableRole tags: - setting @@ -1618,6 +1722,8 @@ paths: properties: name: type: string + cluster: + $ref: "#/components/schemas/DatatableRoleCluster" responses: "200": description: the created role @@ -5351,7 +5457,7 @@ paths: type: string resource_type: type: string - enum: [postgres, instance] + enum: [postgres, instance, external_instance] resource_path: type: string governing_workspace_id: @@ -34048,9 +34154,55 @@ components: - ducklake - datatable + ExternalInstancePgSetupStep: + type: object + required: [name, status, message] + properties: + name: + type: string + status: + type: string + enum: [ok, warning, error] + message: + type: string + + ExternalInstancePgSetupReport: + type: object + required: [success, finished_at, steps] + properties: + success: + type: boolean + description: no step failed; warnings leave it true + finished_at: + type: string + format: date-time + steps: + type: array + items: + $ref: "#/components/schemas/ExternalInstancePgSetupStep" + + ExternalInstancePgStatus: + type: object + required: [configured, database_count] + properties: + configured: + type: boolean + database_count: + type: integer + last_setup: + $ref: "#/components/schemas/ExternalInstancePgSetupReport" + + DatatableRoleCluster: + type: string + description: >- + The Windmill-managed Postgres cluster a data table role is a login on: Windmill's own + (behind `instance` data tables) or the external instance cluster (behind + `external_instance` ones). Defaults to `instance`. + enum: [instance, external_instance] + InstanceDatatableRole: type: object - required: [id, name, enabled] + required: [id, name, enabled, cluster] properties: id: type: string @@ -34058,6 +34210,8 @@ components: type: string enabled: type: boolean + cluster: + $ref: "#/components/schemas/DatatableRoleCluster" DatatableRoleTenants: type: object @@ -34079,8 +34233,10 @@ components: supported: type: boolean description: >- - Whether this data table can be put under roles at all. Only one backed by the - instance database can: a role is a login on that cluster. + Whether this data table can be put under roles at all. Only one on a database Windmill + manages can: a role is a login on that database's cluster. + cluster: + $ref: "#/components/schemas/DatatableRoleCluster" permissioned: type: boolean default_role: @@ -34392,7 +34548,10 @@ components: type: array items: type: string - description: Workspaces that reference this database via a ducklake catalog or datatable database with resource_type 'instance'. Computed at request time, not persisted. + description: Workspaces that reference this database through a ducklake catalog or a datatable database of the kind being listed — 'instance' for the instance databases endpoint, 'external_instance' for the external cluster one. Computed at request time, not persisted, and only returned to superadmins. + workspace_id: + type: string + description: The workspace a member created this database for as a fork copy. Only that workspace can import into it or point a fork at it. NewSqsTrigger: type: object @@ -36400,6 +36559,7 @@ components: - postgresql - mysql - instance + - external_instance resource_path: type: string required: @@ -36463,6 +36623,7 @@ components: enum: - postgresql - instance + - external_instance resource_path: type: string required: diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index e95fadfe66..96fa7ceb63 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -76,6 +76,7 @@ bitflags.workspace = true once_cell.workspace = true phf.workspace = true tokio-postgres.workspace = true +postgres-protocol.workspace = true postgres-native-tls.workspace = true native-tls.workspace = true diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index a9195055ac..f355e733cb 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -6,22 +6,67 @@ * LICENSE-AGPL for a copy of the license. */ -//! The instance's data table role catalog. +//! The instance's data table role catalogs. //! -//! A data table role is a real Postgres login role on the Windmill cluster, named exactly as the -//! user named it, shared by every instance database. Windmill decides who may ask for a role (the -//! per-data-table tenant lists in [`crate::workspaces`]); Postgres decides what the role may then -//! touch. The catalog here is only the first half's vocabulary plus the cluster provisioning. +//! A data table role is a real Postgres login role on one cluster — Windmill's own, or the external +//! instance cluster — named exactly as the user named it, shared by every database Windmill manages +//! on that cluster. Each cluster has its own catalog: a role exists where it was created and nowhere +//! else. Windmill decides who may ask for a role (the per-data-table tenant lists in +//! [`crate::workspaces`]); Postgres decides what the role may then touch. The catalog here is only +//! the first half's vocabulary plus the cluster provisioning. //! //! Entries are keyed by a generated id so a rename moves nothing else: tenants name the id. use std::collections::BTreeMap; +use serde::{Deserialize, Serialize}; + use crate::{ error::{Error, Result}, + workspaces::DataTableCatalogResourceType, DB, }; +/// The cluster a role catalog belongs to. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DatatableRoleCluster { + /// Windmill's own Postgres, behind `instance` data tables. + #[default] + Instance, + /// The external instance cluster, behind `external_instance` data tables. + ExternalInstance, +} + +impl DatatableRoleCluster { + pub fn as_str(self) -> &'static str { + match self { + Self::Instance => "instance", + Self::ExternalInstance => "external_instance", + } + } + + pub fn parse(value: &str) -> Result { + match value { + "instance" => Ok(Self::Instance), + "external_instance" => Ok(Self::ExternalInstance), + other => Err(Error::BadRequest(format!( + "Unknown data table role cluster '{other}': expected instance or external_instance" + ))), + } + } + + /// The cluster whose roles a data table on `kind` can use. `None` for a resource-backed one, + /// which is never under roles. + pub fn of(kind: DataTableCatalogResourceType) -> Option { + match kind { + DataTableCatalogResourceType::Instance => Some(Self::Instance), + DataTableCatalogResourceType::ExternalInstance => Some(Self::ExternalInstance), + DataTableCatalogResourceType::Postgresql => None, + } + } +} + /// The connection every data table resolved to before roles existed (`custom_instance_user`). It /// owns every pre-existing object, so it is a reserved name rather than a catalog entry: never /// created, renamed or dropped. @@ -164,28 +209,42 @@ pub async fn lock_instance_databases_governance<'a>( /// need the names — but callers MUST NOT let `pwd` reach a response, a log line, an audit record /// or an export. Nothing about who may call it: the credential is the whole risk, and `Debug` is /// hand-written to redact it for the same reason. -pub async fn read_role_catalog(db: &DB) -> Result { - crate::datatable_roles_oss::read_role_catalog(db).await +pub async fn read_role_catalog( + db: &DB, + cluster: DatatableRoleCluster, +) -> Result { + crate::datatable_roles_oss::read_role_catalog(db, cluster).await } /// As [`read_role_catalog`], reading inside the caller's transaction so the value is the one /// [`lock_role_catalog`] is protecting. Same disclosure contract. pub async fn read_role_catalog_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + cluster: DatatableRoleCluster, ) -> Result { - crate::datatable_roles_oss::read_role_catalog_tx(tx).await + crate::datatable_roles_oss::read_role_catalog_tx(tx, cluster).await } -/// Record a role, in the caller's transaction so it commits with the `CREATE ROLE` it describes. +/// The cluster a role belongs to, or `None` if no role has this id. +pub async fn role_cluster( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + id: &str, +) -> Result> { + crate::datatable_roles_oss::role_cluster(tx, id).await +} + +/// Record a role, in the caller's transaction. On Windmill's own cluster that commits it with the +/// `CREATE ROLE` it describes; on the external cluster the role already exists by then. /// /// Authorization: writes a generated Postgres credential. Callers MUST restrict this to superadmin /// paths and MUST hold [`lock_role_catalog`] on `tx`. pub async fn insert_role_catalog_entry( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, id: &str, + cluster: DatatableRoleCluster, role: &InstanceDatatableRole, ) -> Result<()> { - crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, role).await + crate::datatable_roles_oss::insert_role_catalog_entry(tx, id, cluster, role).await } /// Update a role's recorded name, login flag and password. Same contract as @@ -215,7 +274,7 @@ pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Res .find(|(_, role)| role.name == name) .ok_or_else(|| { Error::NotFound(format!( - "'{name}' is not a data table role of this instance. Defined roles: {}.", + "'{name}' is not a data table role of this database's cluster. Defined roles: {}.", catalog .values() .map(|r| r.name.as_str()) @@ -231,70 +290,90 @@ pub fn role_id_by_name<'a>(catalog: &'a DatatableRoleCatalog, name: &str) -> Res Ok(entry.0.as_str()) } -/// Every instance database the registry knows about. Role provisioning has to reach all of them: -/// a role that cannot `CONNECT` to a database is refused by Postgres before any grant matters. +/// Every database Windmill manages on `cluster`. Role provisioning has to reach all of them: a role +/// that cannot `CONNECT` to a database is refused by Postgres before any grant matters. /// -/// Authorization: checks nothing, and names every instance database across all workspaces. Callers +/// Authorization: checks nothing, and names every managed database across all workspaces. Callers /// MUST be superadmin-gated or keep the names server-side; never return them to a workspace caller. -pub async fn registered_instance_databases(db: &DB) -> Result> { - crate::datatable_roles_oss::registered_instance_databases(db).await +pub async fn registered_instance_databases( + db: &DB, + cluster: DatatableRoleCluster, +) -> Result> { + crate::datatable_roles_oss::registered_instance_databases(db, cluster).await } -/// `CONNECT` on `dbname` for every enabled role, and none for `PUBLIC`. Run at role creation, at -/// database creation, and lazily whenever an instance data table is administered, so a database -/// provisioned before a role existed is repaired rather than left silently unreachable. +/// `CONNECT` on `dbname` for every enabled role of `cluster`, and none for `PUBLIC`. Run at role +/// creation, at database creation, and lazily whenever a managed data table is administered, so a +/// database provisioned before a role existed is repaired rather than left silently unreachable. /// /// Authorization: rewrites a database's ACL with the server's own credentials and checks nothing. /// Callers MUST have authorized administration of `dbname` — superadmin, or an admin of the /// workspace governing a data table on it. -pub async fn converge_connect_grants(db: &DB, dbname: &str) -> Result<()> { - crate::datatable_roles_oss::converge_connect_grants(db, dbname).await +pub async fn converge_connect_grants( + db: &DB, + cluster: DatatableRoleCluster, + dbname: &str, +) -> Result<()> { + crate::datatable_roles_oss::converge_connect_grants(db, cluster, dbname).await } -/// As [`converge_connect_grants`], with a catalog the caller already read. Same contract. +/// As [`converge_connect_grants`], with the catalog of `cluster` the caller already read. Same +/// contract. pub async fn converge_connect_grants_with( db: &DB, + cluster: DatatableRoleCluster, dbname: &str, catalog: &DatatableRoleCatalog, ) -> Result<()> { - crate::datatable_roles_oss::converge_connect_grants_with(db, dbname, catalog).await + crate::datatable_roles_oss::converge_connect_grants_with(db, cluster, dbname, catalog).await } -/// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user`, and `CONNECT` on -/// every registered database. No privileges beyond that — an admin grants them through SQL or the -/// ACL editor. +/// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user` on `cluster`. No +/// privileges beyond that — an admin grants them through SQL or the ACL editor. +/// +/// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external +/// cluster is another server: the role is created there before `tx` commits, and callers MUST drop +/// it again ([`drop_datatable_role`]) if `tx` then fails to commit. /// /// Authorization: creates a cluster-wide Postgres login. Callers MUST restrict this to superadmin -/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. -pub async fn create_instance_role( +/// paths, and MUST hold [`lock_role_catalog`] on `tx`. +pub async fn create_datatable_role( + db: &DB, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + cluster: DatatableRoleCluster, name: &str, password: &str, ) -> Result<()> { - crate::datatable_roles_oss::create_instance_role(tx, name, password).await + crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await } /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin -/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. -pub async fn set_instance_role_login( +/// paths, and MUST hold [`lock_role_catalog`] on `tx`. +pub async fn set_datatable_role_login( + db: &DB, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + cluster: DatatableRoleCluster, name: &str, enabled: bool, ) -> Result<()> { - crate::datatable_roles_oss::set_instance_role_login(tx, name, enabled).await + crate::datatable_roles_oss::set_datatable_role_login(db, tx, cluster, name, enabled).await } -/// A rename discards an md5-hashed password, so the caller has to hand over a fresh one. +/// A rename discards an md5-hashed password, so the caller has to hand over a fresh one. On the +/// external cluster the rename lands before `tx` commits, and callers MUST rename it back if `tx` +/// then fails to commit. /// /// Authorization: renames a cluster-wide Postgres login. Callers MUST restrict this to superadmin -/// paths, and MUST hold [`lock_role_catalog`] on the same transaction. -pub async fn rename_instance_role( +/// paths, and MUST hold [`lock_role_catalog`] on `tx`. +pub async fn rename_datatable_role( + db: &DB, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + cluster: DatatableRoleCluster, from: &str, to: &str, password: &str, ) -> Result<()> { - crate::datatable_roles_oss::rename_instance_role(tx, from, to, password).await + crate::datatable_roles_oss::rename_datatable_role(db, tx, cluster, from, to, password).await } /// A role owning anything in any database blocks its own `DROP ROLE`, and both its objects and the @@ -302,8 +381,9 @@ pub async fn rename_instance_role( /// registry. An unreachable database aborts the whole delete: dropping the role while one database /// still holds objects owned by it leaves those objects owned by a numeric OID nobody can name. /// -/// Each pass runs as the instance's own Postgres user rather than `custom_instance_user`, which -/// owns the databases and can therefore revoke a grant whoever made it. `custom_instance_user` +/// Each pass runs as the cluster's administrator rather than `custom_instance_user`: on Windmill's +/// own cluster the instance's Postgres user, on the external one its configured admin login. Both +/// own the databases and can therefore revoke a grant whoever made it. `custom_instance_user` /// could only undo what it granted itself, so a privilege planted by an operator in psql — the /// ordinary way privileges reach a role — would survive and block the drop. /// @@ -311,16 +391,19 @@ pub async fn rename_instance_role( /// MUST restrict this to superadmin paths, and MUST hold [`lock_role_catalog`] on `tx`. /// /// The per-database passes open their own connections and cannot join `tx`; the lock is what keeps -/// a concurrent mutation out while they run. Only the final `DROP ROLE` is on `tx`, so it commits -/// or rolls back with the catalog write that forgets the role. Those passes commit as they go, so -/// callers MUST have disabled the role in an earlier committed transaction: a failure part-way -/// then leaves a disabled role to retry, not an enabled one already stripped in some databases. -pub async fn drop_instance_role( +/// a concurrent mutation out while they run. On Windmill's own cluster only the final `DROP ROLE` +/// is on `tx`, so it commits or rolls back with the catalog write that forgets the role; on the +/// external cluster it runs there, and tolerates a role already gone so a retry after a failed +/// commit can finish. The passes commit as they go, so callers MUST have disabled the role in an +/// earlier committed transaction: a failure part-way then leaves a disabled role to retry, not an +/// enabled one already stripped in some databases. +pub async fn drop_datatable_role( db: &DB, tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + cluster: DatatableRoleCluster, name: &str, ) -> Result<()> { - crate::datatable_roles_oss::drop_instance_role(db, tx, name).await + crate::datatable_roles_oss::drop_datatable_role(db, tx, cluster, name).await } #[cfg(test)] diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index 2a3f9dda48..a97a936827 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -24,12 +24,12 @@ pub fn datatable_roles_unavailable() -> Error { #[cfg(all(feature = "private", feature = "enterprise"))] pub(crate) use crate::datatable_roles_ee::{ can_use_datatable_role, can_use_datatable_role_in_governing_workspace, converge_connect_grants, - converge_connect_grants_with, create_instance_role, delete_role_catalog_entry, - drop_instance_role, ensure_can_use_datatable_role, ensure_datatable_admin_access, + converge_connect_grants_with, create_datatable_role, delete_role_catalog_entry, + drop_datatable_role, ensure_can_use_datatable_role, ensure_datatable_admin_access, ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere, insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx, - registered_instance_databases, rename_instance_role, resolve_datatable_role_connection, - set_instance_role_login, update_role_catalog_entry, + registered_instance_databases, rename_datatable_role, resolve_datatable_role_connection, + role_cluster, set_datatable_role_login, update_role_catalog_entry, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -39,7 +39,7 @@ pub(crate) use ce::*; mod ce { use super::datatable_roles_unavailable as unavailable; use crate::{ - datatable_roles::{DatatableRoleCatalog, InstanceDatatableRole}, + datatable_roles::{DatatableRoleCatalog, DatatableRoleCluster, InstanceDatatableRole}, db::AuthedRef, error::Result, workspaces::{ @@ -50,17 +50,31 @@ mod ce { type Tx<'a> = sqlx::Transaction<'a, sqlx::Postgres>; - pub(crate) async fn read_role_catalog(_db: &DB) -> Result { + pub(crate) async fn read_role_catalog( + _db: &DB, + _cluster: DatatableRoleCluster, + ) -> Result { Err(unavailable()) } - pub(crate) async fn read_role_catalog_tx(_tx: &mut Tx<'_>) -> Result { + pub(crate) async fn read_role_catalog_tx( + _tx: &mut Tx<'_>, + _cluster: DatatableRoleCluster, + ) -> Result { + Err(unavailable()) + } + + pub(crate) async fn role_cluster( + _tx: &mut Tx<'_>, + _id: &str, + ) -> Result> { Err(unavailable()) } pub(crate) async fn insert_role_catalog_entry( _tx: &mut Tx<'_>, _id: &str, + _cluster: DatatableRoleCluster, _role: &InstanceDatatableRole, ) -> Result<()> { Err(unavailable()) @@ -78,43 +92,57 @@ mod ce { Err(unavailable()) } - pub(crate) async fn registered_instance_databases(_db: &DB) -> Result> { + pub(crate) async fn registered_instance_databases( + _db: &DB, + _cluster: DatatableRoleCluster, + ) -> Result> { Err(unavailable()) } - /// Nothing to converge: with no roles to admit, an instance database keeps the `CONNECT` - /// grants it was created with, `PUBLIC`'s included, as it did before roles existed. - pub(crate) async fn converge_connect_grants(_db: &DB, _dbname: &str) -> Result<()> { + /// Nothing to converge: with no roles to admit, a managed database keeps the `CONNECT` grants + /// it was created with, as it did before roles existed. + pub(crate) async fn converge_connect_grants( + _db: &DB, + _cluster: DatatableRoleCluster, + _dbname: &str, + ) -> Result<()> { Ok(()) } /// As [`converge_connect_grants`]. pub(crate) async fn converge_connect_grants_with( _db: &DB, + _cluster: DatatableRoleCluster, _dbname: &str, _catalog: &DatatableRoleCatalog, ) -> Result<()> { Ok(()) } - pub(crate) async fn create_instance_role( + pub(crate) async fn create_datatable_role( + _db: &DB, _tx: &mut Tx<'_>, + _cluster: DatatableRoleCluster, _name: &str, _password: &str, ) -> Result<()> { Err(unavailable()) } - pub(crate) async fn set_instance_role_login( + pub(crate) async fn set_datatable_role_login( + _db: &DB, _tx: &mut Tx<'_>, + _cluster: DatatableRoleCluster, _name: &str, _enabled: bool, ) -> Result<()> { Err(unavailable()) } - pub(crate) async fn rename_instance_role( + pub(crate) async fn rename_datatable_role( + _db: &DB, _tx: &mut Tx<'_>, + _cluster: DatatableRoleCluster, _from: &str, _to: &str, _password: &str, @@ -122,12 +150,18 @@ mod ce { Err(unavailable()) } - pub(crate) async fn drop_instance_role(_db: &DB, _tx: &mut Tx<'_>, _name: &str) -> Result<()> { + pub(crate) async fn drop_datatable_role( + _db: &DB, + _tx: &mut Tx<'_>, + _cluster: DatatableRoleCluster, + _name: &str, + ) -> Result<()> { Err(unavailable()) } pub(crate) async fn ensure_instance_db_grant_options_unchecked( _db: &DB, + _cluster: DatatableRoleCluster, _dbname: &str, ) -> Result<()> { Err(unavailable()) diff --git a/backend/windmill-common/src/external_instance_pg.rs b/backend/windmill-common/src/external_instance_pg.rs new file mode 100644 index 0000000000..c0b6203e40 --- /dev/null +++ b/backend/windmill-common/src/external_instance_pg.rs @@ -0,0 +1,456 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The external Postgres cluster behind `external_instance` data tables and Ducklake catalogs. +//! +//! Windmill administers that cluster itself, logged in as the user in +//! [`EXTERNAL_INSTANCE_PG_SETTING`]. It creates `custom_instance_user` and +//! `custom_instance_replication_user` there, with passwords it generates and keeps in +//! [`EXTERNAL_INSTANCE_PG_STATE_SETTING`]. They share their names with the roles on Windmill's own +//! cluster, but they are different roles with different passwords. +//! +//! The cluster may hold data Windmill did not create. Two Windmill instances sharing one is not +//! supported: each would keep resetting the passwords the other depends on. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; + +use crate::{ + error::{Error, Result}, + global_settings::{EXTERNAL_INSTANCE_PG_SETTING, EXTERNAL_INSTANCE_PG_STATE_SETTING}, + instance_config::{CustomInstanceDb, ExternalInstancePg}, + DB, +}; + +/// What Windmill keeps about the external cluster. Server-managed and hidden: never part of the +/// instance config, never readable by an agent worker. No `Debug`: it carries live passwords. +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct ExternalInstancePgState { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_pwd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replication_pwd: Option, + /// The databases Windmill created on the cluster. It only ever drops one of these. + #[serde(default)] + pub databases: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_setup: Option, + /// The cluster ([`external_instance_pg_address`]) the last successful setup converged. Databases + /// are only created on a cluster setup succeeded on: the passwords above exist as soon as setup + /// first runs, whether or not the cluster accepted them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub set_up_for: Option, +} + +/// What identifies the cluster a configuration points at. Other fields (admin login, sslmode) can +/// change without it becoming another cluster. +pub fn external_instance_pg_address(config: &ExternalInstancePg) -> String { + format!( + "{}:{}", + config.host.trim().to_lowercase(), + config.port.unwrap_or(5432) + ) +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ExternalInstancePgSetupReport { + /// No step failed. Warnings leave it true. + pub success: bool, + pub finished_at: chrono::DateTime, + pub steps: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ExternalInstancePgSetupStep { + pub name: String, + pub status: SetupStepStatus, + pub message: String, +} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SetupStepStatus { + Ok, + Warning, + Error, +} + +/// The status the settings page shows without running anything. +#[derive(Serialize, Debug)] +pub struct ExternalInstancePgStatus { + pub configured: bool, + pub database_count: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_setup: Option, +} + +/// Authorization: returns the cluster's admin password and checks nothing. Callers MUST be +/// superadmin or an internal server path. +pub(crate) async fn read_external_instance_pg_config<'c>( + executor: impl sqlx::PgExecutor<'c>, +) -> Result> { + let value = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + EXTERNAL_INSTANCE_PG_SETTING + ) + .fetch_optional(executor) + .await?; + value + .map(|v| { + serde_json::from_value(v).map_err(|e| { + Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_SETTING}: {e}")) + }) + }) + .transpose() +} + +/// Authorization: returns the passwords Windmill generated on the cluster and checks nothing. +/// Callers MUST be superadmin or an internal server path. +pub(crate) async fn read_external_instance_pg_state<'c>( + executor: impl sqlx::PgExecutor<'c>, +) -> Result { + let value = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + EXTERNAL_INSTANCE_PG_STATE_SETTING + ) + .fetch_optional(executor) + .await?; + match value { + None => Ok(ExternalInstancePgState::default()), + Some(v) => serde_json::from_value(v).map_err(|e| { + Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_STATE_SETTING}: {e}")) + }), + } +} + +/// Authorization: reads the hidden cluster state and checks nothing. Callers MUST be superadmin. +pub async fn external_instance_pg_status(db: &DB) -> Result { + let configured = read_external_instance_pg_config(db).await?.is_some(); + let state = read_external_instance_pg_state(db).await?; + Ok(ExternalInstancePgStatus { + configured, + database_count: state.databases.len(), + last_setup: state.last_setup, + }) +} + +/// The databases Windmill created on the external cluster, without the passwords kept beside them. +/// +/// Authorization: names every database across all workspaces, and the workspace each fork copy is +/// reserved for, and checks nothing. Callers MUST be superadmin or an internal authorization or +/// lifecycle path that does not return the names to a workspace caller. +pub async fn external_instance_databases(db: &DB) -> Result> { + Ok(read_external_instance_pg_state(db).await?.databases) +} + +/// The workspaces whose data tables or Ducklake catalogs name each database on the external cluster, +/// and the forks whose Ducklake metadata schemas there are still waiting to be dropped: those rows +/// outlive a settings change, and cleanup cannot drop a schema in a database that is gone. A row +/// whose schema is already dropped only waits on object storage, which needs no database. +/// +/// Authorization: reads every workspace's settings and checks nothing. Callers MUST be superadmin +/// or an internal lifecycle path. +pub async fn external_instance_database_usages<'c>( + db: impl sqlx::PgExecutor<'c>, +) -> Result>> { + let rows = sqlx::query_as::<_, (String, String)>( + "SELECT ws.workspace_id, entry->'database'->>'resource_path' + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' + ELSE '{}'::jsonb END + ) AS dt(k, entry) + WHERE entry->'database'->>'resource_type' = 'external_instance' + AND entry->'database'->>'resource_path' IS NOT NULL + UNION ALL + SELECT ws.workspace_id, entry->'catalog'->>'resource_path' + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object' + THEN ws.ducklake->'ducklakes' + ELSE '{}'::jsonb END + ) AS dl(k, entry) + WHERE entry->'catalog'->>'resource_type' = 'external_instance' + AND entry->'catalog'->>'resource_path' IS NOT NULL + UNION ALL + SELECT workspace_id, substring(catalog FROM length('external_instance:') + 1) + FROM fork_ducklake_namespace + WHERE catalog LIKE 'external\\_instance:%' AND NOT schema_dropped", + ) + .fetch_all(db) + .await?; + let mut usages: BTreeMap> = BTreeMap::new(); + for (workspace_id, dbname) in rows { + usages.entry(dbname).or_default().insert(workspace_id); + } + Ok(usages) +} + +/// Refuse to unset the cluster while Windmill still has databases or data table roles on it, or a +/// workspace still points at one: every data table there would stop resolving, and every role +/// would be a login nothing can drop any more. Allowed on every edition, so a +/// downgraded instance can still clear a setting it no longer uses. +async fn ensure_external_instance_pg_removable(conn: &mut sqlx::PgConnection) -> Result<()> { + ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")).await +} + +/// Refuse while Windmill has databases or data table roles on the cluster, or a workspace points +/// at one of its databases. `before` finishes the sentence saying what to do first. +async fn ensure_external_instance_pg_unused( + conn: &mut sqlx::PgConnection, + before: &str, +) -> Result<()> { + let state = read_external_instance_pg_state(&mut *conn).await?; + let usages = external_instance_database_usages(&mut *conn).await?; + let roles = sqlx::query_scalar::<_, String>( + "SELECT name FROM datatable_role WHERE cluster = 'external_instance' ORDER BY name", + ) + .fetch_all(&mut *conn) + .await?; + if state.databases.is_empty() && usages.is_empty() && roles.is_empty() { + return Ok(()); + } + let mut held = vec![]; + if !(state.databases.is_empty() && usages.is_empty()) { + let names = state + .databases + .keys() + .chain(usages.keys()) + .collect::>() + .into_iter() + .cloned() + .collect::>() + .join(", "); + held.push(format!("databases in use ({names})")); + } + if !roles.is_empty() { + held.push(format!("data table roles ({})", roles.join(", "))); + } + Err(Error::BadRequest(format!( + "The external instance cluster still holds {}. Drop them and repoint the data tables and \ + Ducklake catalogs using them before {before}.", + held.join(" and ") + ))) +} + +/// Refuse a workspace setting that newly names an `external_instance` database on an edition +/// without them. +pub fn ensure_external_instance_available() -> Result<()> { + crate::external_instance_pg_oss::ensure_external_instance_available() +} + +/// The connection an `external_instance` database resolves to: `custom_instance_user`, or the +/// replication user, on the external cluster. +/// +/// Authorization: returns live credentials and checks nothing. Callers MUST have authorized access +/// to the data table that names `dbname`. +pub async fn external_instance_connection_unchecked( + db: &DB, + dbname: &str, + replication: bool, +) -> Result { + crate::external_instance_pg_oss::external_instance_connection_unchecked(db, dbname, replication) + .await +} + +/// Create `dbname` on the external cluster and register it. Refuses a name already taken there, +/// whoever took it. +/// +/// Runs on `tx`, which it takes [`lock_external_instance_pg_state`] on: the registration lands when +/// the caller commits. Take that lock before any database governance lock, as saves do. +/// +/// Authorization: checks nothing. Callers MUST be superadmin, or be cloning a data table they may +/// fork into a `wm_fork_` database. +pub async fn create_external_instance_database_unchecked( + db: &DB, + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + dbname: &str, + tag: &str, + for_workspace: Option<&str>, +) -> Result<()> { + crate::external_instance_pg_oss::create_external_instance_database_unchecked( + db, + tx, + dbname, + tag, + for_workspace, + ) + .await +} + +/// Drop `dbname` from the external cluster: only a database Windmill registered creating, and still +/// carries the mark it set there. Refused while anything uses it +/// ([`crate::workspaces::managed_database_uses`]), except the `exempt` data table entry: the fork +/// copy being cleaned up. +/// +/// Runs on `tx`, like [`create_external_instance_database_unchecked`]: the database is gone at +/// once, its registry entry when the caller commits. +/// +/// Authorization: checks nothing. Callers MUST be superadmin, or be deleting the fork that owns +/// this `wm_fork_` database. +pub async fn drop_external_instance_database_unchecked( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + dbname: &str, + exempt: Option<(&str, &str)>, +) -> Result<()> { + crate::external_instance_pg_oss::drop_external_instance_database_unchecked(tx, dbname, exempt) + .await +} + +/// Serializes everything that changes which databases exist on the external cluster, or which data +/// tables name them: setup, creates, drops, and data table saves. Held until `tx` ends. +pub async fn lock_external_instance_pg_state( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, +) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext($1))") + .bind(EXTERNAL_INSTANCE_PG_STATE_SETTING) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Refuse a data table naming `dbname` unless Windmill created it on the external cluster. Takes +/// the lock drops take, so none can remove the database before `tx`, which saves the data table, +/// commits. +/// +/// Authorization: its refusal says whether Windmill created a database of that name, which is +/// instance-wide knowledge. Callers MUST have authorized the caller as superadmin first. +pub async fn ensure_external_instance_database_registered( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + dbname: &str, +) -> Result<()> { + lock_external_instance_pg_state(tx).await?; + if read_external_instance_pg_state(&mut **tx) + .await? + .databases + .contains_key(dbname) + { + return Ok(()); + } + Err(Error::BadRequest(format!( + "Windmill did not create a database named '{dbname}' on the external instance cluster. \ + Create it from the instance settings first." + ))) +} + +/// Write [`EXTERNAL_INSTANCE_PG_SETTING`]: `None`, null or an empty string unsets it. Every writer +/// of global settings goes through this for that key — the per-key and bulk endpoints as well as +/// the declarative sync — instead of writing the row itself. +/// +/// The checks and the write share one transaction holding [`lock_external_instance_pg_state`]. A +/// check taken outside it could pass while a database create still reads the old cluster, which +/// would then register a database there after the setting names another one. +/// +/// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config +/// sync, which applies what the operator deployed. +pub async fn write_external_instance_pg_setting( + db: &DB, + value: Option<&serde_json::Value>, +) -> Result<()> { + let value = match value { + None | Some(serde_json::Value::Null) => None, + Some(serde_json::Value::String(s)) if s.trim().is_empty() => None, + Some(value) => Some(value), + }; + let mut tx = db.begin().await?; + lock_external_instance_pg_state(&mut tx).await?; + // Every check runs on this transaction's own connection: it holds the advisory lock, and + // taking a second connection from the pool while other writers queue on that lock is how a + // small pool deadlocks. + match value { + None => { + ensure_external_instance_pg_removable(&mut tx).await?; + sqlx::query("DELETE FROM global_settings WHERE name = $1") + .bind(EXTERNAL_INSTANCE_PG_SETTING) + .execute(&mut *tx) + .await?; + } + Some(value) => { + crate::external_instance_pg_oss::validate_external_instance_pg_setting(value)?; + ensure_external_instance_pg_not_repointed(&mut tx, value).await?; + sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()", + ) + .bind(EXTERNAL_INSTANCE_PG_SETTING) + .bind(value) + .execute(&mut *tx) + .await?; + } + } + tx.commit().await?; + tracing::info!( + "{} global setting {EXTERNAL_INSTANCE_PG_SETTING}", + if value.is_some() { "Set" } else { "Unset" } + ); + Ok(()) +} + +/// [`write_external_instance_pg_setting`] for a settings diff: writes the key if the diff touches +/// it, and takes it out of the diff so the generic apply does not write it again. +/// +/// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config +/// sync, which applies what the operator deployed. +pub async fn write_external_instance_pg_from_diff( + db: &DB, + diff: &mut crate::instance_config::SettingsDiff, +) -> Result<()> { + if let Some(value) = diff.upserts.remove(EXTERNAL_INSTANCE_PG_SETTING) { + write_external_instance_pg_setting(db, Some(&value)).await?; + } + if let Some(i) = diff + .deletes + .iter() + .position(|k| k == EXTERNAL_INSTANCE_PG_SETTING) + { + diff.deletes.remove(i); + write_external_instance_pg_setting(db, None).await?; + } + Ok(()) +} + +/// Refuse pointing the setting at another host or port while databases or data table roles live on +/// the current one. Data tables name databases, and the role catalog names logins, not clusters, so +/// both would silently resolve to whatever the new cluster holds under the same names. Other fields +/// (admin login, sslmode) may change freely. +async fn ensure_external_instance_pg_not_repointed( + conn: &mut sqlx::PgConnection, + value: &serde_json::Value, +) -> Result<()> { + let Some(current) = read_external_instance_pg_config(&mut *conn).await? else { + return Ok(()); + }; + let Ok(desired) = serde_json::from_value::(value.clone()) else { + return Ok(()); + }; + if external_instance_pg_address(¤t) == external_instance_pg_address(&desired) { + return Ok(()); + } + ensure_external_instance_pg_unused( + conn, + &format!("pointing {EXTERNAL_INSTANCE_PG_SETTING} at another cluster"), + ) + .await +} + +/// Converge the external cluster on the configured login: check what it can do, create or update +/// Windmill's two roles with the stored passwords, and report anything that would get in the way. +/// With `rotate_passwords`, generate new passwords first. Safe to run again; running it again is +/// how a failed rotation is repaired. +/// +/// Authorization: administers the external cluster with its admin credentials and checks nothing. +/// Callers MUST be superadmin. +pub async fn setup_external_instance_pg_unchecked( + db: &DB, + rotate_passwords: bool, +) -> Result { + crate::external_instance_pg_oss::setup_external_instance_pg_unchecked(db, rotate_passwords) + .await +} diff --git a/backend/windmill-common/src/external_instance_pg_oss.rs b/backend/windmill-common/src/external_instance_pg_oss.rs new file mode 100644 index 0000000000..93b6fc63dd --- /dev/null +++ b/backend/windmill-common/src/external_instance_pg_oss.rs @@ -0,0 +1,82 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the external instance cluster comes from: the enterprise implementation, or a refusal. +//! `private` alone is not that edition: community builds carry it. + +use crate::error::Error; + +pub fn external_instance_pg_unavailable() -> Error { + Error::BadRequest( + "External instance databases are a Windmill Enterprise Edition feature".to_string(), + ) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::external_instance_pg_ee::{ + create_external_instance_database_unchecked, drop_external_instance_database_unchecked, + external_instance_connection_unchecked, setup_external_instance_pg_unchecked, + validate_external_instance_pg_setting, +}; + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) fn ensure_external_instance_available() -> crate::error::Result<()> { + Ok(()) +} + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use super::external_instance_pg_unavailable as unavailable; + use crate::{ + error::Result, external_instance_pg::ExternalInstancePgSetupReport, PgDatabase, DB, + }; + + pub(crate) fn validate_external_instance_pg_setting(_value: &serde_json::Value) -> Result<()> { + Err(unavailable()) + } + + pub(crate) fn ensure_external_instance_available() -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn setup_external_instance_pg_unchecked( + _db: &DB, + _rotate_passwords: bool, + ) -> Result { + Err(unavailable()) + } + + pub(crate) async fn external_instance_connection_unchecked( + _db: &DB, + _dbname: &str, + _replication: bool, + ) -> Result { + Err(unavailable()) + } + + pub(crate) async fn create_external_instance_database_unchecked( + _db: &DB, + _tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + _dbname: &str, + _tag: &str, + _for_workspace: Option<&str>, + ) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn drop_external_instance_database_unchecked( + _tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + _dbname: &str, + _exempt: Option<(&str, &str)>, + ) -> Result<()> { + Err(unavailable()) + } +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 7306c5c59a..9137dc364e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -57,6 +57,11 @@ pub const SAML_METADATA_SETTING: &str = "saml_metadata"; pub const SMTP_SETTING: &str = "smtp_settings"; pub const TEAMS_SETTING: &str = "teams"; pub const INDEXER_SETTING: &str = "indexer_settings"; +pub const EXTERNAL_INSTANCE_PG_SETTING: &str = "external_instance_pg"; +/// Turns off Windmill's own Postgres as a data table and Ducklake substrate. Absent means on, +/// which is what every instance that predates the setting expects. +pub const INSTANCE_PG_DISABLED_SETTING: &str = "instance_pg_disabled"; +pub const EXTERNAL_INSTANCE_PG_STATE_SETTING: &str = "external_instance_pg_state"; pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result"; pub const UNIQUE_ID_SETTING: &str = "uid"; @@ -448,6 +453,9 @@ pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[ // resolve datatable connections through the dedicated datatable endpoints, never these. "custom_instance_pg_databases", "custom_instance_replication_pwd", + // The external cluster's admin login, and the passwords Windmill generated on it. + EXTERNAL_INSTANCE_PG_SETTING, + EXTERNAL_INSTANCE_PG_STATE_SETTING, ]; /// Whether an agent worker may read the given global setting over HTTP. diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 264916d522..fd790547b6 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -354,6 +354,8 @@ pub struct GlobalSettings { pub ducklake_settings: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_instance_pg_databases: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub external_instance_pg: Option, // Opaque settings (EE-private structs or no clear schema) #[serde(skip_serializing_if = "Option::is_none")] @@ -815,6 +817,9 @@ pub struct CustomInstanceDb { pub error: Option, #[serde(skip_serializing_if = "Option::is_none")] pub tag: Option, + /// The workspace a member created this fork copy for. Absent when a superadmin created it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_id: Option, } /// Setup log entries for a custom instance database. @@ -841,6 +846,36 @@ pub struct CustomInstanceDbLogs { pub user_connect: String, } +// --------------------------------------------------------------------------- +// External instance PG cluster +// --------------------------------------------------------------------------- + +/// The external Postgres cluster Windmill manages for `external_instance` data tables and Ducklake +/// catalogs. `user` logs in as the cluster's administrator: it needs `CREATEDB` and `CREATEROLE`. +/// `dbname` is only where that login connects to run cluster-wide statements. +/// +/// Every field defaults rather than being required: this deserializes as part of the whole +/// instance config, and one malformed row must not make every other setting unreadable. The +/// write path and every use reject an incomplete value instead. +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] +pub struct ExternalInstancePg { + #[serde(default)] + pub host: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, + #[serde(default)] + pub user: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dbname: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sslmode: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_certificate_pem: Option, +} + // --------------------------------------------------------------------------- // Autoscaling (worker config) // --------------------------------------------------------------------------- @@ -977,6 +1012,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "ducklake_settings", "custom_instance_pg_databases", "custom_instance_replication_pwd", + "external_instance_pg_state", "uid", "rsa_keys", "jwt_secret", @@ -1002,6 +1038,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ // Server-only (written by setup/refresh via direct SQL), never operator-authored — // hidden so the config machinery can't read, rewrite, or drop it. "custom_instance_replication_pwd", + // Same for the passwords and database registry Windmill keeps for the external cluster. + "external_instance_pg_state", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. @@ -1013,6 +1051,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[ "license_key", "ducklake_user_pg_pwd", "custom_instance_replication_pwd", + "external_instance_pg_state", "pip_index_url", "pip_extra_index_url", "npm_config_registry", @@ -1038,6 +1077,7 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[ &["secret_key", "serviceAccountKey", "accessKey"], ), ("custom_instance_pg_databases", &["user_pwd"]), + ("external_instance_pg", &["password"]), ("github_enterprise_app", &["private_key"]), ]; @@ -1379,7 +1419,8 @@ pub async fn sync_global_settings_declarative( crate::global_settings::parse_max_token_expiration_days(desired.get(max_expiration_key)) .map_err(|e| anyhow::anyhow!("{max_expiration_key}: {e}"))?; - let diff = diff_global_settings(current, desired, ApplyMode::Replace); + let mut diff = diff_global_settings(current, desired, ApplyMode::Replace); + crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff).await?; apply_settings_diff(db, &diff).await?; Ok(()) @@ -1512,6 +1553,10 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> { resolve_env_option(&mut pg.user_pwd)?; } + if let Some(pg) = &mut settings.external_instance_pg { + resolve_env_option(&mut pg.password)?; + } + Ok(()) } @@ -2495,39 +2540,33 @@ mod tests { } #[test] - fn custom_instance_replication_pwd_is_isolated_from_config() { - // The replication-role password is server-only: written by setup/refresh via direct - // SQL, never operator-authored. It must stay out of the declarative config surface - // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it. - assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd")); - assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd")); - assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd")); + fn server_generated_db_passwords_are_isolated_from_config() { + // These hold passwords the server generates: written by setup/refresh via direct SQL, + // never operator-authored. They must stay out of the declarative config surface + // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop them. + for key in [ + "custom_instance_replication_pwd", + "external_instance_pg_state", + ] { + assert!(HIDDEN_SETTINGS.contains(&key), "{key}"); + assert!(PROTECTED_SETTINGS.contains(&key), "{key}"); + assert!(SENSITIVE_SETTINGS.contains(&key), "{key}"); - // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted. - let mut desired = BTreeMap::new(); - desired.insert( - "custom_instance_replication_pwd".to_string(), - serde_json::json!("attacker-set"), - ); - let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge); - assert!( - diff.upserts.is_empty(), - "hidden setting must not be upserted" - ); + // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted. + let mut desired = BTreeMap::new(); + desired.insert(key.to_string(), serde_json::json!("attacker-set")); + let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge); + assert!(diff.upserts.is_empty(), "{key} must not be upserted"); - // A current value is never deleted by a Replace that omits it. - let mut current = BTreeMap::new(); - current.insert( - "custom_instance_replication_pwd".to_string(), - serde_json::json!("live"), - ); - let diff = diff_global_settings(¤t, &BTreeMap::new(), ApplyMode::Replace); - assert!( - !diff - .deletes - .contains(&"custom_instance_replication_pwd".to_string()), - "hidden setting must not be deleted" - ); + // A current value is never deleted by a Replace that omits it. + let mut current = BTreeMap::new(); + current.insert(key.to_string(), serde_json::json!("live")); + let diff = diff_global_settings(¤t, &BTreeMap::new(), ApplyMode::Replace); + assert!( + !diff.deletes.contains(&key.to_string()), + "{key} must not be deleted" + ); + } } #[test] diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 47a7ea5b2a..0a1b0974d5 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -58,6 +58,10 @@ pub mod ee_oss; pub mod email_ee; pub mod email_oss; pub mod error; +pub mod external_instance_pg; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod external_instance_pg_ee; +pub mod external_instance_pg_oss; pub mod external_ip; #[cfg(feature = "private")] pub mod feature_usage_ee; @@ -1115,7 +1119,13 @@ impl PgDatabase { if err_str.contains("password authentication failed for user") && err_str.contains("custom_instance_user") { - if let Some(db) = main_db { + // The external instance cluster has a `custom_instance_user` of its own, whose + // password setup manages. Rotating the local one would break every instance + // data table and fix nothing. + let local = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + let on_local_cluster = local.host == self.host + && local.port.unwrap_or(5432) == self.port.unwrap_or(5432); + if let Some(db) = main_db.filter(|_| on_local_cluster) { tracing::warn!( "custom_instance_user password auth failed, refreshing and retrying..." ); @@ -1665,13 +1675,41 @@ pub async fn instance_database_users( } /// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. +/// +/// Authorization: drops any instance database but Windmill's own and checks nothing. Callers MUST +/// be superadmin, or have established the caller may drop this one — a fork's owner cleaning up +/// its own copy that nothing else uses. pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { drop_custom_instance_database_on(&mut *db.acquire().await?, dbname).await } +/// [`drop_custom_instance_database`] leaving its registry entry, for a caller holding row locks in +/// a transaction: the registry write has to go through that transaction, as waiting on another +/// connection for a lock the transaction's own peers hold is a deadlock Postgres cannot see. Same +/// authorization contract. +pub async fn drop_custom_instance_database_keep_entry(db: &DB, dbname: &str) -> error::Result<()> { + drop_instance_database_keep_entry_on(&mut *db.acquire().await?, dbname).await +} + async fn drop_custom_instance_database_on( conn: &mut sqlx::PgConnection, dbname: &str, +) -> error::Result<()> { + let dbname = dbname.trim(); + drop_instance_database_keep_entry_on(&mut *conn, dbname).await?; + // Always remove from global_settings + sqlx::query!( + r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, + dbname + ) + .execute(&mut *conn) + .await?; + Ok(()) +} + +async fn drop_instance_database_keep_entry_on( + conn: &mut sqlx::PgConnection, + dbname: &str, ) -> error::Result<()> { let dbname = dbname.trim(); validate_dbname(dbname)?; @@ -1718,14 +1756,6 @@ async fn drop_custom_instance_database_on( tracing::info!("Database '{}' does not exist, skipping drop", dbname); } - // Always remove from global_settings - sqlx::query!( - r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, - dbname - ) - .execute(&mut *conn) - .await?; - Ok(()) } @@ -1750,26 +1780,30 @@ pub(crate) fn instance_db_grants(dbname: &str) -> String { ) } -/// Re-apply [`instance_db_grants`] to an instance database provisioned before data table roles -/// existed, whose grants carry no grant option. Connects as the instance's own Postgres user — -/// the database and `public` schema owner — since only it can hand out an option it holds. +/// Re-apply [`instance_db_grants`] to a managed database provisioned before data table roles +/// existed, whose grants carry no grant option. Connects as the cluster's administrator — the +/// database and `public` schema owner — since only it can hand out an option it holds. /// -/// Authorization: reaches an instance database with the server's own credentials and checks +/// Authorization: reaches a managed database with the server's own credentials and checks /// nothing. Callers MUST have authorized administration of `dbname` — superadmin, or an admin of /// the workspace governing a data table on it. pub async fn ensure_instance_db_grant_options_unchecked( db: &DB, + cluster: crate::datatable_roles::DatatableRoleCluster, dbname: &str, ) -> error::Result<()> { - crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, dbname).await + crate::datatable_roles_oss::ensure_instance_db_grant_options_unchecked(db, cluster, dbname) + .await } /// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings. -/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). +/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). `for_workspace` +/// is the workspace a member creates a fork copy for; see [`ensure_fork_database_available_to`]. pub async fn create_custom_instance_database( db: &DB, dbname: &str, tag: &str, + for_workspace: Option<&str>, ) -> error::Result<()> { let dbname = dbname.trim(); validate_dbname(dbname)?; @@ -1799,7 +1833,7 @@ pub async fn create_custom_instance_database( // Nothing names a database that failed past this point, and its name blocks the retry: drop it // rather than leave it behind. - if let Err(e) = finish_custom_instance_database(db, dbname, tag).await { + if let Err(e) = finish_custom_instance_database(db, dbname, tag, for_workspace).await { match drop_unused_instance_database(db, dbname).await { Ok(Cleanup::InUse(users)) => tracing::warn!( "Kept '{dbname}' after failing to set it up: workspaces {} use it", @@ -1820,7 +1854,13 @@ pub async fn create_custom_instance_database( // A data table role can only reach a database it may CONNECT to, and PUBLIC's default CONNECT // would otherwise let every role in regardless of what this instance defines. Best-effort: a // failure here leaves the database usable as `admin`, and the next role change repairs it. - if let Err(e) = crate::datatable_roles::converge_connect_grants(db, dbname).await { + if let Err(e) = crate::datatable_roles::converge_connect_grants( + db, + crate::datatable_roles::DatatableRoleCluster::Instance, + dbname, + ) + .await + { tracing::warn!("Could not set CONNECT grants on instance database '{dbname}': {e}"); } @@ -1829,7 +1869,12 @@ pub async fn create_custom_instance_database( } /// Grant `custom_instance_user` its privileges on a database just created, and register it. -async fn finish_custom_instance_database(db: &DB, dbname: &str, tag: &str) -> error::Result<()> { +async fn finish_custom_instance_database( + db: &DB, + dbname: &str, + tag: &str, + for_workspace: Option<&str>, +) -> error::Result<()> { // Grant permissions to custom_instance_user let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; let new_pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds }; @@ -1856,7 +1901,8 @@ async fn finish_custom_instance_database(db: &DB, dbname: &str, tag: &str) -> er }, "success": true, "error": null, - "tag": tag + "tag": tag, + "workspace_id": for_workspace, }); sqlx::query!( r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#, @@ -1864,6 +1910,75 @@ async fn finish_custom_instance_database(db: &DB, dbname: &str, tag: &str) -> er ) .execute(db) .await?; + + Ok(()) +} + +/// The system's CA bundle file, for libpq clients that cannot take `sslrootcert=system`: that value +/// needs libpq 16, and verify-full only. +pub fn system_ca_bundle() -> Option { + std::env::var_os("SSL_CERT_FILE") + .map(std::path::PathBuf::from) + .into_iter() + .chain( + [ + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/cert.pem", + "/etc/ssl/ca-bundle.pem", + ] + .map(std::path::PathBuf::from), + ) + .find(|path| path.is_file()) +} + +/// Refuse a workspace member writing a fork copy into, or pointing a fork at, the managed database +/// `dbname` of `kind`, unless `w_id` created it for that ([`create_custom_instance_database`], or +/// its external instance counterpart) and nothing uses it yet. The `wm_fork_` prefix is no +/// authorization: every database of a cluster answers to the same `custom_instance_user`, so a name +/// is all it takes to reach another workspace's copy. +/// +/// Runs on `conn`: its callers hold a transaction with the fork lock while they check, and a +/// second connection taken from the pool under it is how a small pool deadlocks. +/// +/// Authorization: reads the global registries and every workspace's settings, and names other +/// workspaces in its refusal. Callers MUST have authorized `w_id` for the caller first — a member +/// of it forking or importing there — and MUST NOT call it on a workspace the caller is not in. +pub async fn ensure_fork_database_available_to( + conn: &mut sqlx::PgConnection, + kind: workspaces::DataTableCatalogResourceType, + dbname: &str, + w_id: &str, +) -> error::Result<()> { + let created_for = match kind { + workspaces::DataTableCatalogResourceType::ExternalInstance => { + external_instance_pg::read_external_instance_pg_state(&mut *conn) + .await? + .databases + .remove(dbname) + .and_then(|entry| entry.workspace_id) + } + _ => sqlx::query_scalar::<_, Option>( + "SELECT value->'databases'->$1->>'workspace_id' FROM global_settings + WHERE name = 'custom_instance_pg_databases'", + ) + .bind(dbname) + .fetch_optional(&mut *conn) + .await? + .flatten(), + }; + if created_for.as_deref() != Some(w_id) { + return Err(Error::BadRequest(format!( + "Database '{dbname}' was not created for a fork of workspace '{w_id}'" + ))); + } + let uses = workspaces::managed_database_uses(conn, kind, dbname, None).await?; + if !uses.is_empty() { + return Err(Error::BadRequest(format!( + "Database '{dbname}' is already in use: {}", + uses.join(", ") + ))); + } Ok(()) } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index b285332b7d..724743dce9 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1542,6 +1542,47 @@ pub enum DataTableCatalogResourceType { #[strum(serialize = "postgres")] Postgresql, Instance, + /// On the external instance cluster ([`crate::external_instance_pg`]). Enterprise Edition. + #[serde(rename = "external_instance")] + #[strum(serialize = "external_instance")] + ExternalInstance, +} + +impl DataTableCatalogResourceType { + /// A database Windmill created and administers, on its own cluster or the external one, as + /// opposed to one a user brought as a resource. + pub fn is_windmill_managed(self) -> bool { + matches!(self, Self::Instance | Self::ExternalInstance) + } +} + +/// Refuse a new use of Windmill's own Postgres as a data table or Ducklake substrate where an +/// operator turned it off, and on the managed cloud, which never had it. Entries already on it +/// keep resolving: this gates what a save may newly name, not what runs. +pub async fn ensure_instance_pg_available<'c>(executor: impl sqlx::PgExecutor<'c>) -> Result<()> { + if *crate::worker::CLOUD_HOSTED { + return Err(Error::BadRequest( + "Windmill's own database cannot back a data table or Ducklake catalog on Windmill Cloud" + .to_string(), + )); + } + let disabled = sqlx::query_scalar::<_, Option>( + "SELECT value FROM global_settings WHERE name = $1", + ) + .bind(crate::global_settings::INSTANCE_PG_DISABLED_SETTING) + .fetch_optional(executor) + .await? + .flatten() + .is_some_and(|v| v.as_bool().unwrap_or(false)); + if disabled { + return Err(Error::BadRequest( + "Windmill's own database is disabled as a data table and Ducklake substrate on this \ + instance. Use the external instance cluster, or turn it back on in the instance \ + settings." + .to_string(), + )); + } + Ok(()) } /// Build a self-teaching error for an unresolved `datatable://` reference. @@ -1621,14 +1662,82 @@ pub struct GoverningDatatable { pub governor: Option, } +/// Everything still using the Windmill-managed database `dbname`, one description per use: data +/// table entries naming it, fork entries pointing at those, Ducklake catalogs on it, and fork +/// Ducklake metadata schemas there that cleanup has not dropped yet. `exempt` is the one data table +/// entry, `(workspace_id, name)`, the caller is about to stop using it through; pointers at that +/// entry still count, since dropping the database would leave them resolving to nothing. +/// +/// Authorization: reads every workspace's settings and checks nothing. Callers MUST only turn the +/// answer into a refusal for someone allowed to administer `dbname`. +pub async fn managed_database_uses( + conn: &mut sqlx::PgConnection, + kind: DataTableCatalogResourceType, + dbname: &str, + exempt: Option<(&str, &str)>, +) -> Result> { + let (exempt_workspace, exempt_name) = exempt.unzip(); + Ok(sqlx::query_scalar::<_, String>( + "WITH entries AS ( + SELECT ws.workspace_id::text AS workspace_id, dt.key AS name, dt.value + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object' + THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt + ), naming AS ( + SELECT workspace_id, name FROM entries + WHERE value->'database'->>'resource_type' = $1 + AND value->'database'->>'resource_path' = $2 + ) + SELECT format('data table ''%s'' in workspace ''%s''', name, workspace_id) FROM naming + WHERE $3::text IS NULL OR NOT (workspace_id = $3 AND name = $4) + UNION ALL + SELECT format('data table ''%s'' in workspace ''%s'', which points at the one in ''%s''', + e.name, e.workspace_id, n.workspace_id) + FROM entries e JOIN naming n + ON e.value->'reference'->>'workspace_id' = n.workspace_id + AND e.value->'reference'->>'datatable' = n.name + UNION ALL + SELECT format('Ducklake ''%s'' in workspace ''%s''', dl.key, ws.workspace_id) + FROM workspace_settings ws + CROSS JOIN LATERAL jsonb_each( + CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object' + THEN ws.ducklake->'ducklakes' ELSE '{}'::jsonb END) dl + WHERE dl.value->'catalog'->>'resource_type' = $1 + AND dl.value->'catalog'->>'resource_path' = $2 + UNION ALL + SELECT format('the Ducklake namespace of fork ''%s'', not cleaned up yet', workspace_id) + FROM fork_ducklake_namespace + WHERE catalog = $1 || ':' || $2 AND NOT schema_dropped + ORDER BY 1", + ) + .bind(kind.as_ref()) + .bind(dbname) + .bind(exempt_workspace) + .bind(exempt_name) + .fetch_all(&mut *conn) + .await?) +} + +/// Held by fork cleanup of `w_id`'s data tables and by forking `w_id`, which can hand the new fork +/// pointers at them, so a pointer cannot appear between cleanup's check and its drop. +pub async fn lock_fork_datatables(conn: &mut sqlx::PgConnection, w_id: &str) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork_datatables:' || $1))") + .bind(w_id) + .execute(&mut *conn) + .await?; + Ok(()) +} + impl GoverningDatatable { - /// Backed by the Windmill instance's own Postgres, which is the only substrate data table - /// roles apply to. - pub fn is_instance(&self) -> bool { + /// The Windmill-managed cluster whose data table roles this entry can use. `None` for a + /// resource-backed one: roles are logins Windmill creates, and it creates none on a host a + /// workspace admin chose. + pub fn role_cluster(&self) -> Option { self.datatable .database .as_ref() - .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance) + .and_then(|d| crate::datatable_roles::DatatableRoleCluster::of(d.resource_type)) } /// The workspace whose admins administer the data table and whose members its tenants are. @@ -1776,20 +1885,19 @@ pub async fn resolve_workspace_governing_datatables( clone, )), (None, Some(governed_by)) => { - let (governor_ws, governor_name) = - (governed_by.workspace_id.clone(), governed_by.datatable.clone()); + let (governor_ws, governor_name) = ( + governed_by.workspace_id.clone(), + governed_by.datatable.clone(), + ); let clone = clone.or(Some((ws, name, datatable))); next.push((i, governor_ws, governor_name, clone)); } (None, None) => resolved.push(( i, match clone { - None => GoverningDatatable { - workspace_id: ws, - name, - datatable, - governor: None, - }, + None => { + GoverningDatatable { workspace_id: ws, name, datatable, governor: None } + } Some((clone_ws, clone_name, mut clone_datatable)) => { clone_datatable.permissions = datatable.permissions; GoverningDatatable { @@ -1830,7 +1938,8 @@ pub async fn resolve_workspace_governing_datatables( } /// Build the `admin` connection for a governing entry: `custom_instance_user` for an instance -/// database, the user's own resource for a BYO-postgres one. +/// database, on Windmill's cluster or the external one; the user's own resource for a BYO-postgres +/// one. async fn resolve_datatable_connection_unchecked( db: &DB, governing: &GoverningDatatable, @@ -1841,7 +1950,16 @@ async fn resolve_datatable_connection_unchecked( .database .as_ref() .expect("a governing entry owns a database"); - if database.resource_type == DataTableCatalogResourceType::Instance { + if database.resource_type == DataTableCatalogResourceType::ExternalInstance { + let pg_creds = crate::external_instance_pg::external_instance_connection_unchecked( + db, + &database.resource_path, + replication, + ) + .await?; + serde_json::to_value(&pg_creds) + .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e))) + } else if database.resource_type == DataTableCatalogResourceType::Instance { let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; pg_creds.dbname = database.resource_path.clone(); if replication { @@ -1879,8 +1997,31 @@ pub async fn get_datatable_resource_from_db_unchecked( w_id: &str, name: &str, ) -> Result { + Ok(get_datatable_connection_and_kind_unchecked(db, w_id, name) + .await? + .0) +} + +/// As [`get_datatable_resource_from_db_unchecked`], also reporting the kind of database Windmill +/// manages behind it, `None` for a user resource. One resolution answers both: a caller reading the +/// kind separately can be handed one kind's connection and the other kind's checks by a save +/// landing between the two, and the entry a pointer lands on is another workspace's to change. +/// +/// Same authorization contract: the connection reaches every role. +pub async fn get_datatable_connection_and_kind_unchecked( + db: &DB, + w_id: &str, + name: &str, +) -> Result<(serde_json::Value, Option)> { let governing = resolve_governing_datatable(db, w_id, name).await?; - resolve_datatable_connection_unchecked(db, &governing, false).await + let kind = governing + .datatable + .database + .as_ref() + .map(|d| d.resource_type) + .filter(|kind| kind.is_windmill_managed()); + let connection = resolve_datatable_connection_unchecked(db, &governing, false).await?; + Ok((connection, kind)) } /// Same as [`get_datatable_resource_from_db_unchecked`] but for postgres trigger @@ -2385,6 +2526,10 @@ pub enum DucklakeCatalogResourceType { Postgresql, Mysql, Instance, + /// On the external instance cluster ([`crate::external_instance_pg`]). Enterprise Edition. + #[serde(rename = "external_instance")] + #[strum(serialize = "external_instance")] + ExternalInstance, } #[derive(Deserialize, Serialize)] @@ -2912,7 +3057,16 @@ async fn ducklake_conn_data( let ducklake = serde_json::from_value::(ducklake)?; let catalog_resource = - if ducklake.catalog.resource_type == DucklakeCatalogResourceType::Instance { + if ducklake.catalog.resource_type == DucklakeCatalogResourceType::ExternalInstance { + let pg_creds = crate::external_instance_pg::external_instance_connection_unchecked( + db, + &ducklake.catalog.resource_path, + false, + ) + .await?; + serde_json::to_value(&pg_creds) + .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))? + } else if ducklake.catalog.resource_type == DucklakeCatalogResourceType::Instance { let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; pg_creds.dbname = ducklake.catalog.resource_path.clone(); pg_creds.user = Some("custom_instance_user".to_string()); @@ -3293,6 +3447,14 @@ async fn register_fork_ducklake_namespace( { return Ok(()); } + let mut tx = db.begin().await?; + // A row naming an external database counts as a use of it. Written under the lock a drop takes, + // and only while the database is still registered, so a drop cannot slip in between the + // settings this attach resolved and the row that protects the database. + if let Some(dbname) = catalog.strip_prefix("external_instance:") { + crate::external_instance_pg::ensure_external_instance_database_registered(&mut tx, dbname) + .await?; + } sqlx::query!( "INSERT INTO fork_ducklake_namespace (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path) @@ -3307,9 +3469,10 @@ async fn register_fork_ducklake_namespace( &storage_ref, data_path, ) - .execute(db) + .execute(&mut *tx) .await .map_err(|e| Error::internal_err(format!("registering fork ducklake namespace: {e:#}")))?; + tx.commit().await?; let mut locations = FORK_DUCKLAKE_REGISTERED .get(w_id) .filter(|(_, exp)| *exp > now) diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 943703472f..d3bd976f21 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -1475,6 +1475,7 @@ pub async fn do_duckdb( &job.id, client, &mut hidden_passwords, + job_dir, ) .await?, ); @@ -1490,13 +1491,19 @@ pub async fn do_duckdb( &mut hidden_passwords, &job.workspace_id, materialize.as_ref().map(|(_, m)| m.asset_path.as_str()), + job_dir, ) .await? { probe_blocks.extend(q); - } else if let Some(q) = - transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job) - .await? + } else if let Some(q) = transform_attach_datatable( + &query_block, + conn, + &mut hidden_passwords, + job, + job_dir, + ) + .await? { probe_blocks.extend(q); } else { @@ -1552,6 +1559,7 @@ pub async fn do_duckdb( &job.id, client, &mut hidden_passwords, + job_dir, ) .await?, ); @@ -1567,13 +1575,19 @@ pub async fn do_duckdb( &mut hidden_passwords, &job.workspace_id, materialize.as_ref().map(|(_, m)| m.asset_path.as_str()), + job_dir, ) .await? { v.extend(ducklake_query); - } else if let Some(datatable_query) = - transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job) - .await? + } else if let Some(datatable_query) = transform_attach_datatable( + &query_block, + conn, + &mut hidden_passwords, + job, + job_dir, + ) + .await? { v.extend(datatable_query); } else { @@ -2240,11 +2254,80 @@ fn parse_attach_db_resource<'a>(query: &'a str) -> Option Result { +/// The verification a DuckDB postgres attach keeps, as its libpq `sslmode` and `sslrootcert`. +/// +/// Attaches have always turned verify-ca and verify-full into `require`, which resources rely on. +/// A connection that explicitly refuses invalid certificates — the external instance cluster's — +/// keeps its mode instead: under `require` its shared password would go to whichever server +/// answers. DuckDB's libpq takes one root file, so it gets the system bundle plus the configured +/// certificate, written in the job directory: a resource's certificate is workspace-controlled, so +/// a file per distinct one has to go with the job rather than pile up on the worker. +fn pg_attach_verification<'a>( + res: &'a PgDatabase, + job_dir: &str, +) -> Result> { + let mode = match res.sslmode.as_deref() { + // The rule every other Postgres connection uses, so a resource that verifies elsewhere is + // not quietly downgraded here. + Some(mode @ ("verify-ca" | "verify-full")) if !res.verify_mode_skips_verification() => mode, + _ => return Ok(None), + }; + let bundle = windmill_common::system_ca_bundle() + .map(std::fs::read_to_string) + .transpose() + .map_err(|e| Error::ExecutionErr(format!("Failed to read the system CA bundle: {e}")))? + .unwrap_or_default(); + let pem = res.root_certificate_pem.as_deref().unwrap_or_default(); + if bundle.is_empty() && pem.is_empty() { + return Err(Error::ExecutionErr(format!( + "sslmode {mode} needs a root certificate, and this worker has no system CA bundle" + ))); + } + let roots = format!("{bundle}\n{pem}\n"); + use sha2::Digest; + let path = std::path::Path::new(job_dir).join(format!( + "pg_roots_{}.pem", + hex::encode(&sha2::Sha256::digest(roots.as_bytes())[..8]) + )); + // The job's modules are written in this directory first, so whatever already sits at this path + // may be caller-supplied: trusting it would let the caller pick the CA. Replace it, and + // `create_new` refuses to write through anything recreated there. + let write_roots = || -> std::io::Result<()> { + match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e), + _ => {} + } + use std::io::Write; + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path)? + .write_all(roots.as_bytes()) + }; + write_roots() + .map_err(|e| Error::ExecutionErr(format!("Failed to write root certificates: {e}")))?; + Ok(Some((mode, path))) +} + +fn pg_attach_uri(res: &PgDatabase, job_dir: &str) -> Result { + let uri = res.to_uri(); + let Some((mode, roots)) = pg_attach_verification(res, job_dir)? else { + return Ok(uri); + }; + let base = uri.strip_suffix("?sslmode=require").ok_or_else(|| { + Error::internal_err("unexpected sslmode in a postgres connection URI".to_string()) + })?; + Ok(format!( + "{base}?sslmode={mode}&sslrootcert={}", + urlencoding::encode(&roots.to_string_lossy()) + )) +} + +fn format_attach_db_conn_str(db_resource: Value, db_type: &str, job_dir: &str) -> Result { let s = match db_type.to_lowercase().as_str() { "postgres" | "postgresql" => { let res: PgDatabase = serde_json::from_value(db_resource)?; - res.to_uri() + pg_attach_uri(&res, job_dir)? } #[cfg(feature = "mysql")] "mysql" => { @@ -2316,6 +2399,7 @@ async fn transform_attach_db_resource_query( job_id: &Uuid, client: &AuthedClient, hidden_passwords: &mut Arc>>, + job_dir: &str, ) -> Result> { let db_resource: Value = client .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string())) @@ -2323,8 +2407,14 @@ async fn transform_attach_db_resource_query( if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) { hidden_passwords.lock().unwrap().push(pwd.to_string()); } - db_resource_to_attach_statements(db_resource, parsed.name, parsed.db_type, parsed.extra_args) - .await + db_resource_to_attach_statements( + db_resource, + parsed.name, + parsed.db_type, + parsed.extra_args, + job_dir, + ) + .await } async fn db_resource_to_attach_statements( @@ -2332,11 +2422,12 @@ async fn db_resource_to_attach_statements( ident_name: &str, db_type: &str, extra_args: Option<&str>, + job_dir: &str, ) -> Result> { // Escape single quotes: the connection string is built from resource fields // (host/db/user/password) and embedded in a single-quoted DuckDB literal, so an // unescaped quote in any field would otherwise break out of the ATTACH statement. - let conn_str = format_attach_db_conn_str(db_resource, db_type)?.replace('\'', "''"); + let conn_str = format_attach_db_conn_str(db_resource, db_type, job_dir)?.replace('\'', "''"); let attach_str = format!( "ATTACH '{}' as {} (TYPE {}{});", conn_str, @@ -2359,6 +2450,7 @@ async fn transform_attach_ducklake( hidden_passwords: &mut Arc>>, w_id: &str, materialize_target: Option<&str>, + job_dir: &str, ) -> Result>> { lazy_static::lazy_static! { static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'ducklake(://[^':]+)?'\s*AS\s+([^ ;]+)\s*(\([^)]*\))?").unwrap(); @@ -2391,7 +2483,9 @@ async fn transform_attach_ducklake( format!(", {}", user_extra_args) }; let db_type = match ducklake.catalog.resource_type { - DucklakeCatalogResourceType::Instance => "postgres", + DucklakeCatalogResourceType::Instance | DucklakeCatalogResourceType::ExternalInstance => { + "postgres" + } _ => ducklake.catalog.resource_type.as_ref(), }; @@ -2407,7 +2501,7 @@ async fn transform_attach_ducklake( // single-quoted DuckDB literals below, so an unescaped quote in a resource // field would break out of the ATTACH statement. let db_conn_str = - format_attach_db_conn_str(ducklake.catalog_resource, db_type)?.replace('\'', "''"); + format_attach_db_conn_str(ducklake.catalog_resource, db_type, job_dir)?.replace('\'', "''"); let storage = ducklake .storage .storage @@ -2462,6 +2556,7 @@ async fn transform_attach_ducklake( defer, materialize_target, hidden_passwords, + job_dir, )?); } Ok(Some(statements)) @@ -2495,6 +2590,7 @@ fn fork_defer_statements( defer: &windmill_common::workspaces::DucklakeForkDefer, materialize_target: Option<&str>, hidden_passwords: &mut Arc>>, + job_dir: &str, ) -> Result> { let mut stmts = vec![]; if defer.ancestors.is_empty() { @@ -2507,12 +2603,13 @@ fn fork_defer_statements( hidden_passwords.lock().unwrap().push(pwd.to_string()); } let db_type = match a.catalog.resource_type { - DucklakeCatalogResourceType::Instance => "postgres", + DucklakeCatalogResourceType::Instance + | DucklakeCatalogResourceType::ExternalInstance => "postgres", _ => a.catalog.resource_type.as_ref(), }; stmts.push(get_attach_db_install_str(db_type)?.to_string()); - let conn_str = - format_attach_db_conn_str(a.catalog_resource.clone(), db_type)?.replace('\'', "''"); + let conn_str = format_attach_db_conn_str(a.catalog_resource.clone(), db_type, job_dir)? + .replace('\'', "''"); let storage = a .storage .storage @@ -2631,6 +2728,7 @@ async fn transform_attach_datatable( conn: &Connection, hidden_passwords: &mut Arc>>, job: &MiniPulledJob, + job_dir: &str, ) -> Result>> { let Some(attached) = parse_attach_datatable(query) else { return Ok(None); @@ -2673,6 +2771,7 @@ async fn transform_attach_datatable( Ok(Some(pg_secret_attach_statements( db_resource, attached.alias, + job_dir, )?)) } @@ -2693,17 +2792,32 @@ fn datatable_secret_name(alias: &str) -> String { /// ATTACH a datatable's postgres database through a DuckDB TEMPORARY SECRET holding /// the connection parameters; only sslmode and options ride in the ATTACH string. -fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result> { +fn pg_secret_attach_statements( + db_resource: Value, + alias_name: &str, + job_dir: &str, +) -> Result> { let res: PgDatabase = serde_json::from_value(db_resource)?; // Escape single quotes: each field is embedded in a single-quoted DuckDB literal, // so an unescaped quote would break out of the CREATE SECRET statement. let esc = |s: &str| s.replace('\'', "''"); // The postgres secret type has no sslmode parameter, so it goes in the ATTACH // string; only the libpq values PgDatabase::to_uri collapses to are forwarded. - let sslmode = match res.sslmode.as_deref() { - Some("disable") => "disable", - Some("require") | Some("verify-ca") | Some("verify-full") => "require", - _ => "prefer", + let sslmode = match pg_attach_verification(&res, job_dir)? { + // A libpq keyword/value string: the path is quoted for libpq, then for the DuckDB literal. + Some((mode, roots)) => format!( + "{mode} sslrootcert=''{}''", + roots + .to_string_lossy() + .replace('\\', "\\\\") + .replace('\'', "\\''") + ), + None => match res.sslmode.as_deref() { + Some("disable") => "disable", + Some("require") | Some("verify-ca") | Some("verify-full") => "require", + _ => "prefer", + } + .to_string(), }; // Nor an options parameter. The value is quoted for libpq's keyword/value syntax first, // then escaped for the DuckDB literal around it. @@ -2805,6 +2919,76 @@ pub struct Arg { mod tests { use super::*; + #[test] + fn pg_attach_keeps_verification_only_when_required() { + let job_dir = std::env::temp_dir().join(format!("wm-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&job_dir).unwrap(); + let job_dir = job_dir.to_string_lossy().to_string(); + let pg = |sslmode: &str, accept_invalid_certs: Option| PgDatabase { + host: "db.internal".to_string(), + user: Some("custom_instance_user".to_string()), + password: Some("pw".to_string()), + port: None, + sslmode: Some(sslmode.to_string()), + dbname: "dt".to_string(), + root_certificate_pem: Some("-----BEGIN CERTIFICATE-----test".to_string()), + accept_invalid_certs, + use_iam_auth: None, + region: None, + options: None, + }; + let uri = pg_attach_uri(&pg("verify-full", Some(false)), &job_dir).unwrap(); + assert!(uri.contains("?sslmode=verify-full&sslrootcert="), "{uri}"); + let root = urlencoding::decode(uri.split("sslrootcert=").nth(1).unwrap()).unwrap(); + assert!(std::fs::read_to_string(root.as_ref()) + .unwrap() + .contains("-----BEGIN CERTIFICATE-----test")); + // A file a job module planted under the same name is not trusted. + std::fs::write(root.as_ref(), "-----BEGIN CERTIFICATE-----planted").unwrap(); + assert_eq!( + pg_attach_uri(&pg("verify-full", Some(false)), &job_dir).unwrap(), + uri + ); + assert!(!std::fs::read_to_string(root.as_ref()) + .unwrap() + .contains("planted")); + // Every certificate a job attaches keeps its own file: one attach must not evict another's. + for i in 0..40 { + let mut other = pg("verify-full", Some(false)); + other.root_certificate_pem = Some(format!("-----BEGIN CERTIFICATE-----{i}")); + let other = pg_attach_uri(&other, &job_dir).unwrap(); + let path = urlencoding::decode(other.split("sslrootcert=").nth(1).unwrap()).unwrap(); + assert!(std::path::Path::new(path.as_ref()).is_file(), "{path}"); + } + assert!( + std::path::Path::new(root.as_ref()).is_file(), + "the first file is still there" + ); + let external = serde_json::to_value(pg("verify-full", Some(false))).unwrap(); + let attach = &pg_secret_attach_statements(external, "dt", &job_dir).unwrap()[3]; + assert!( + attach.starts_with(&format!( + "ATTACH 'sslmode=verify-full sslrootcert=''{}''", + root + )), + "{attach}" + ); + // A resource carrying a root certificate verifies without opting in, as it does on every + // other Postgres path; one carrying neither keeps the historical downgrade. + assert!(pg_attach_uri(&pg("verify-full", None), &job_dir) + .unwrap() + .contains("?sslmode=verify-full&sslrootcert=")); + let mut bare = pg("verify-full", None); + bare.root_certificate_pem = None; + assert!(pg_attach_uri(&bare, &job_dir) + .unwrap() + .ends_with("?sslmode=require")); + assert!(pg_attach_uri(&pg("require", Some(false)), &job_dir) + .unwrap() + .ends_with("?sslmode=require")); + std::fs::remove_dir_all(&job_dir).unwrap(); + } + #[test] fn attach_datatable_parses_name_and_role() { let reference_of = |q: &str| parse_attach_datatable(q).unwrap().reference; @@ -2956,7 +3140,7 @@ mod tests { let mut defer = test_fork_defer(vec![("orders", false)], vec![]); defer.ancestors[0].extra_args = Some("ENCRYPTED true".to_string()); let mut hp = Arc::new(Mutex::new(vec![])); - let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap(); let attach = stmts .iter() .find(|s| s.starts_with("ATTACH IF NOT EXISTS")) @@ -2980,7 +3164,7 @@ mod tests { vec![], ); let mut hp = Arc::new(Mutex::new(vec![])); - let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap(); let joined = stmts.join("\n"); assert!( joined.contains( @@ -3012,7 +3196,7 @@ mod tests { fn test_fork_defer_statements_shape() { let defer = test_fork_defer(vec![("orders", false), ("dim", true)], vec![]); let mut hp = Arc::new(Mutex::new(vec![])); - let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap(); let joined = stmts.join("\n"); // Ancestor attach: read-only, idempotent, never auto-migrating or auto-creating. assert!(joined.contains("ATTACH IF NOT EXISTS"), "{joined}"); @@ -3038,9 +3222,15 @@ mod tests { // Target currently a defer view → skip its CREATE, drop the view (+ companion). let defer = test_fork_defer(vec![("orders", false)], vec!["orders", "orders_current"]); let mut hp = Arc::new(Mutex::new(vec![])); - let stmts = - fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp) - .unwrap(); + let stmts = fork_defer_statements( + "lake", + "_wm_target", + &defer, + Some("lake/orders"), + &mut hp, + "/tmp", + ) + .unwrap(); let joined = stmts.join("\n"); assert!(!joined.contains("CREATE VIEW"), "{joined}"); assert!( @@ -3055,15 +3245,22 @@ mod tests { // Target already a real table (NOT in fork_views, e.g. after a failed re-run whose // status can't be trusted) → no DROP VIEW, or the job would wedge on a type mismatch. let defer = test_fork_defer(vec![("orders", false)], vec![]); - let stmts = - fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp) - .unwrap(); + let stmts = fork_defer_statements( + "lake", + "_wm_target", + &defer, + Some("lake/orders"), + &mut hp, + "/tmp", + ) + .unwrap(); assert!(!stmts.join("\n").contains("DROP VIEW"), "{stmts:?}"); // Target in a different lake → this lake's defer views are untouched. let defer = test_fork_defer(vec![("orders", false)], vec!["orders"]); let stmts = - fork_defer_statements("lake", "dl", &defer, Some("other/orders"), &mut hp).unwrap(); + fork_defer_statements("lake", "dl", &defer, Some("other/orders"), &mut hp, "/tmp") + .unwrap(); let joined = stmts.join("\n"); assert!( joined.contains("CREATE VIEW IF NOT EXISTS dl.\"orders\""), @@ -3076,7 +3273,7 @@ mod tests { fn test_fork_defer_statements_schema_qualified() { let defer = test_fork_defer(vec![("staging.raw", false)], vec![]); let mut hp = Arc::new(Mutex::new(vec![])); - let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap(); let joined = stmts.join("\n"); assert!( joined.contains("CREATE SCHEMA IF NOT EXISTS dl.\"staging\";"), @@ -3902,7 +4099,7 @@ mod tests { "dbname": "mydb", "sslmode": "require" }); - let result = format_attach_db_conn_str(db_resource, "postgres").unwrap(); + let result = format_attach_db_conn_str(db_resource, "postgres", "/tmp").unwrap(); // Should be in URI format: postgres://user:password@host:port/dbname?sslmode=require assert!(result.starts_with("postgres://")); assert!(result.contains("admin:secret123@localhost:5432/mydb")); @@ -3915,7 +4112,7 @@ mod tests { "host": "db.example.com", "dbname": "production" }); - let result = format_attach_db_conn_str(db_resource, "postgres").unwrap(); + let result = format_attach_db_conn_str(db_resource, "postgres", "/tmp").unwrap(); // Should be in URI format with defaults: postgres://postgres:@host:5432/dbname?sslmode=prefer assert!(result.starts_with("postgres://")); assert!(result.contains("@db.example.com:5432/production")); @@ -3928,7 +4125,7 @@ mod tests { "host": "localhost", "dbname": "test" }); - let result = format_attach_db_conn_str(db_resource, "postgresql").unwrap(); + let result = format_attach_db_conn_str(db_resource, "postgresql", "/tmp").unwrap(); // Should be in URI format (postgresql is treated the same as postgres) assert!(result.starts_with("postgres://")); assert!(result.contains("@localhost:5432/test")); @@ -3945,7 +4142,7 @@ mod tests { "dbname": "wm_datatables", "sslmode": "require" }); - let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap(); + let stmts = pg_secret_attach_statements(db_resource, "dt", "/tmp").unwrap(); assert_eq!(stmts[0], "INSTALL postgres;"); assert_eq!(stmts[1], "LOAD postgres;"); let secret_name = datatable_secret_name("dt"); @@ -3976,7 +4173,7 @@ mod tests { if let Some(s) = input { db_resource["sslmode"] = json!(s); } - let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap(); + let stmts = pg_secret_attach_statements(db_resource, "dt", "/tmp").unwrap(); assert!( stmts[3].starts_with(&format!("ATTACH 'sslmode={expected}'")), "sslmode {input:?} → {}", @@ -3988,7 +4185,7 @@ mod tests { #[test] fn test_pg_secret_attach_statements_options() { let db_resource = json!({ "host": "h", "dbname": "d", "options": r"-c search_path='a\b'" }); - let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap(); + let stmts = pg_secret_attach_statements(db_resource, "dt", "/tmp").unwrap(); let secret_name = datatable_secret_name("dt"); assert_eq!( stmts[3], @@ -4012,7 +4209,7 @@ mod tests { let db_resource = json!({ "project_id": "my-gcp-project" }); - let result = format_attach_db_conn_str(db_resource, "bigquery").unwrap(); + let result = format_attach_db_conn_str(db_resource, "bigquery", "/tmp").unwrap(); assert_eq!(result, "project=my-gcp-project"); } @@ -4021,7 +4218,7 @@ mod tests { let db_resource = json!({ "other_field": "value" }); - let result = format_attach_db_conn_str(db_resource, "bigquery"); + let result = format_attach_db_conn_str(db_resource, "bigquery", "/tmp"); assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("project_id")); } @@ -4029,7 +4226,7 @@ mod tests { #[test] fn test_format_attach_db_conn_str_unsupported_type() { let db_resource = json!({}); - let result = format_attach_db_conn_str(db_resource, "oracle"); + let result = format_attach_db_conn_str(db_resource, "oracle", "/tmp"); assert!(result.is_err()); assert!(result .unwrap_err() @@ -4043,7 +4240,7 @@ mod tests { "host": "localhost", "dbname": "test" }); - let result = format_attach_db_conn_str(db_resource, "POSTGRES").unwrap(); + let result = format_attach_db_conn_str(db_resource, "POSTGRES", "/tmp").unwrap(); // Should be in URI format assert!(result.starts_with("postgres://")); assert!(result.contains("@localhost:5432/test")); @@ -4060,7 +4257,7 @@ mod tests { "database": "app_db", "ssl": true }); - let result = format_attach_db_conn_str(db_resource, "mysql").unwrap(); + let result = format_attach_db_conn_str(db_resource, "mysql", "/tmp").unwrap(); assert!(result.contains("database=app_db")); assert!(result.contains("host=mysql.example.com")); assert!(result.contains("ssl_mode=required")); @@ -4077,7 +4274,7 @@ mod tests { "database": "test", "ssl": false }); - let result = format_attach_db_conn_str(db_resource, "mysql").unwrap(); + let result = format_attach_db_conn_str(db_resource, "mysql", "/tmp").unwrap(); assert!(result.contains("ssl_mode=disabled")); } diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts index 6609d08c50..46ee7b4c39 100644 --- a/cli/src/commands/datatable/datatable.ts +++ b/cli/src/commands/datatable/datatable.ts @@ -111,6 +111,8 @@ const migrateCommand = new Command() ) .action(migrateDown as any); +type DataTableResourceType = "postgresql" | "instance" | "external_instance"; + async function create( opts: GlobalOptions & { resource?: string; force?: boolean }, name?: string, @@ -139,12 +141,12 @@ async function create( const datatables: Record< string, - { database: { resource_type: "postgresql" | "instance"; resource_path?: string } } + { database: { resource_type: DataTableResourceType; resource_path?: string } } > = {}; for (const d of existing) { datatables[d.name] = { database: { - resource_type: d.resource_type as "postgresql" | "instance", + resource_type: d.resource_type as DataTableResourceType, resource_path: d.resource_path ?? undefined, }, }; diff --git a/docs/external-instance-datatables.md b/docs/external-instance-datatables.md new file mode 100644 index 0000000000..acfb93b90a --- /dev/null +++ b/docs/external-instance-datatables.md @@ -0,0 +1,88 @@ +# External instance data tables + +A data table is backed by one of three things: a Postgres resource a workspace brings +(`postgresql`), a database on Windmill's own cluster (`instance`), or a database on a separate +cluster Windmill administers (`external_instance`, Enterprise Edition). The third is what this +document covers; Ducklake catalogs take the same three shapes. + +Windmill administers the external cluster the way it administers its own: it creates and drops +databases there, owns `custom_instance_user` and `custom_instance_replication_user`, and creates +the data table roles of that cluster. It logs in as the admin in the `external_instance_pg` +instance setting, and keeps what it generates in the hidden `external_instance_pg_state` setting. + +## Code + +| Where | What | +|---|---| +| `windmill-common/src/external_instance_pg.rs` | Setting, state, usage accounting, the lifecycle lock, the OSS forwarders | +| `windmill-common/src/external_instance_pg_ee.rs` | Setup, database create and drop, the admin connection | +| `windmill-common/src/datatable_roles.rs` | Per-cluster role catalogs (`DatatableRoleCluster`) | +| `windmill-common/src/workspaces.rs` | Resolution (`resolve_datatable_connection_unchecked`), `managed_database_uses` | +| `windmill-api-settings/src/lib.rs` | `/settings/external_instance_pg/*`, `/settings/datatable_roles` | + +## What holds it together + +- **One lifecycle lock.** `lock_external_instance_pg_state` serializes everything that changes + which databases exist on the cluster or which entries name them: setup, create, drop, data table + and Ducklake saves, external role DDL, and writes to the setting itself. Anything reading the + configuration to reach the cluster reads it under that lock, so a database is never created on + one cluster and registered while the setting names another. +- **Windmill only touches what it made.** Databases it creates carry a comment, and a drop + requires it. The two managed roles and every data table role carry their own comment, and setup + refuses a `custom_instance_user` without it rather than resetting the password of someone else's + role. +- **Creation needs a successful setup.** `set_up_for` records the `host:port` the last successful + setup converged. Creating a database on a cluster that setup has not succeeded on is refused. +- **Nothing is dropped from under a user.** `managed_database_uses` lists every data table naming + a database, every fork pointing at those, every Ducklake catalog on it, and every fork Ducklake + metadata schema still to be dropped. Fork cleanup exempts exactly the entry it is cleaning up. +- **Fork copies belong to a workspace.** `wm_fork_*` is a name, not an authorization: every + database of a cluster answers to the same `custom_instance_user`. The registry records the + workspace a copy was created for, and a member can only import into or fork onto a copy of their + own workspace. +- **Roles are per cluster.** `datatable_role.cluster` splits the catalog, so the same role name can + exist on both clusters. Role names are unique per cluster, as they are in Postgres. + +## Running one locally + +```bash +docker run -d --name wm-external-pg -e POSTGRES_PASSWORD=external -p 5497:5432 postgres:18 \ + -c wal_level=logical +psql "postgresql://postgres:external@127.0.0.1:5497/postgres" \ + -c "CREATE ROLE wm_admin LOGIN PASSWORD 'adminpw' CREATEDB CREATEROLE REPLICATION" +``` + +A non-superuser admin with `CREATEDB` and `CREATEROLE` is the realistic case: managed Postgres +gives nothing more. `REPLICATION` is only needed for Postgres triggers on external data tables. + +Then, as superadmin (`$T` is a token): + +```bash +api=http://localhost:8000/api +curl -s -X POST $api/settings/global/external_instance_pg -H "Authorization: Bearer $T" \ + -H 'Content-Type: application/json' \ + --data '{"value":{"host":"127.0.0.1","port":5497,"user":"wm_admin","password":"adminpw","sslmode":"disable"}}' +curl -s -X POST $api/settings/external_instance_pg/setup -H "Authorization: Bearer $T" \ + -H 'Content-Type: application/json' --data '{}' # report per step +curl -s -X POST $api/settings/external_instance_pg/databases/dt_demo -H "Authorization: Bearer $T" \ + -H 'Content-Type: application/json' --data '{}' +curl -s -X POST $api/w/admins/workspaces/edit_datatable_config -H "Authorization: Bearer $T" \ + -H 'Content-Type: application/json' \ + --data '{"settings":{"datatables":{"demo":{"database":{"resource_type":"external_instance","resource_path":"dt_demo"}}}}}' +``` + +`sslmode` defaults to `verify-full`; `disable` is for a local container only. With `verify-full` +against a server with a private CA, put the CA in `root_certificate_pem` — `pg_dump`, `psql` and +DuckDB attaches all verify against the system trust store plus that certificate. + +Jobs then reach it as any data table: `ATTACH 'datatable://demo' AS d` from DuckDB, or +`datatable://demo` as the database of a PostgreSQL script, with `-- role ` to connect as a +data table role of that cluster. + +Worth knowing while testing: + +- A worker needs the `postgresql` and `duckdb` tags for those jobs + (`update config set config = jsonb_set(config, '{worker_tags}', …) where name = 'worker__default'`). +- DuckDB jobs load `libwindmill_duckdb_ffi_internal.so` by name, so a binary built into its own + `CARGO_TARGET_DIR` needs that library on `LD_LIBRARY_PATH`. +- Setup holds the lifecycle lock for its whole run, so a settings save during it waits. diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 0da63ffbb0..d6740c6c9b 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -23,6 +23,8 @@ import RetentionPeriodOverrides from './instanceSettings/RetentionPeriodOverrides.svelte' import SmtpSettings from './instanceSettings/SmtpSettings.svelte' import SecretBackendConfig from './instanceSettings/SecretBackendConfig.svelte' + import ExternalInstancePgSettings from './instanceSettings/ExternalInstancePgSettings.svelte' + import InstancePgSettings from './instanceSettings/InstancePgSettings.svelte' import GhesAppSettings from './instanceSettings/GhesAppSettings.svelte' import WebhookBaseUrlSetting from './instanceSettings/WebhookBaseUrlSetting.svelte' import WsConnectivityTest from './instanceSettings/WsConnectivityTest.svelte' @@ -43,6 +45,7 @@ openSmtpSettings?: () => void oauths?: Record warning?: string + markSettingSaved?: (key: string) => void } let { @@ -52,7 +55,8 @@ loading = true, openSmtpSettings, oauths, - warning + warning, + markSettingSaved }: Props = $props() const dispatch = createEventDispatcher() @@ -782,10 +786,10 @@ />

    Comma-separated host/IP patterns the proxy still traces but for which it skips - upstream TLS certificate verification. Use for internal endpoints with - self-signed or otherwise untrusted certificates — unlike NO_PROXY above, these - requests stay traced. Same matching as NO_PROXY (example.com matches - subdomains; .example.com matches subdomains only). + upstream TLS certificate verification. Use for internal endpoints with self-signed + or otherwise untrusted certificates — unlike NO_PROXY above, these requests stay + traced. Same matching as NO_PROXY (example.com matches subdomains; + .example.com matches subdomains only).

    @@ -865,6 +869,14 @@ {:else if setting.fieldType == 'secret_backend'} + {:else if setting.fieldType == 'external_instance_pg'} + + {:else if setting.fieldType == 'instance_pg'} + {:else if setting.fieldType == 'github_enterprise_app'} {:else if setting.fieldType == 'webhook_base_url'} diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 0c27a3c7c4..1476b9ad87 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -60,6 +60,14 @@ let requirePreexistingUserForOauth: boolean = $state(false) let initialValues: Record = $state({}) + + /// A setting its own component writes (it has to persist before acting on the cluster) is + /// already saved, so the baseline must move with it: otherwise Discard restores the value it + /// replaced, and a later bulk save sends that stale one back. + function markSettingSaved(key: string) { + initialValues[key] = + $values[key] === undefined ? undefined : JSON.parse(JSON.stringify($values[key])) + } let baseUrlIsFallback = $state(false) // Per-instance OAuth providers (Snowflake, ServiceNow, …): instance name // keyed by provider, used to build their per-instance connect_config URLs. @@ -730,6 +738,7 @@ secret_backend: ['token', 'client_secret', 'secret_access_key'], object_store_cache_config: ['secret_key', 'serviceAccountKey', 'accessKey'], custom_instance_pg_databases: ['user_pwd'], + external_instance_pg: ['password'], rsa_keys: ['private_key'], github_enterprise_app: ['private_key'] } @@ -1194,6 +1203,7 @@ {loading} {setting} {values} + {markSettingSaved} {version} {oauths} /> @@ -1213,6 +1223,7 @@ {loading} {setting} {values} + {markSettingSaved} {version} {oauths} warning={setting.key === 'base_url' && baseUrlIsFallback @@ -1228,6 +1239,7 @@ {@const licenseKeySetting = settings['Core'].find((s) => s.key === 'license_key')} {#if licenseKeySetting} closeDrawer?.()} {loading} @@ -1253,6 +1265,7 @@ {loading} {setting} {values} + {markSettingSaved} {version} {oauths} /> diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 17d5ec9f6e..9701b74e30 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -68,6 +68,8 @@ export interface Setting { | 'otel' | 'otel_tracing_proxy' | 'secret_backend' + | 'external_instance_pg' + | 'instance_pg' | 'github_enterprise_app' | 'webhook_base_url' | 'ws_connectivity' @@ -599,6 +601,25 @@ export const settings: Record = { (Number.isInteger(Number(v)) && Number(v) >= 0 && Number(v) <= 3650) } ], + 'Managed Postgres': [ + { + label: 'External instance', + description: + 'A Postgres cluster Windmill administers for data tables and Ducklake catalogs, instead of its own database. It creates the databases there and manages the roles jobs connect as.', + key: 'external_instance_pg', + fieldType: 'external_instance_pg', + storage: 'setting', + ee_only: '' + }, + { + label: 'Windmill instance', + description: + "Windmill's own database as a data table and Ducklake substrate. On unless turned off here, and never available on cloud.", + key: 'instance_pg_disabled', + fieldType: 'instance_pg', + storage: 'setting' + } + ], 'Object Storage': [ { label: 'Instance object storage', @@ -1309,6 +1330,14 @@ export const instanceSettingsNavigationGroups = [ aiId: 'instance-settings-object-storage', aiDescription: 'Instance object storage settings', isEE: true + }, + { + id: 'managed_postgres', + label: 'Managed Postgres', + aiId: 'instance-settings-managed-postgres', + aiDescription: + 'Postgres substrates Windmill administers for data tables and Ducklake catalogs: its own database and an external cluster', + isEE: true } ] }, @@ -1428,6 +1457,7 @@ export const tabToCategoryMap: Record = { telemetry: 'Telemetry', secret_storage: 'Secret Storage', object_storage: 'Object Storage', + managed_postgres: 'Managed Postgres', jobs: 'Jobs', private_hub: 'Private Hub', github_enterprise_app: 'GitHub App', @@ -1464,6 +1494,7 @@ export const categoryToTabMap: Record = { Telemetry: 'telemetry', 'Secret Storage': 'secret_storage', 'Object Storage': 'object_storage', + 'Managed Postgres': 'managed_postgres', Jobs: 'jobs', 'Private Hub': 'private_hub', 'GitHub App': 'github_enterprise_app', diff --git a/frontend/src/lib/components/instanceSettings/ExternalInstancePgSettings.svelte b/frontend/src/lib/components/instanceSettings/ExternalInstancePgSettings.svelte new file mode 100644 index 0000000000..97b5494f44 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/ExternalInstancePgSettings.svelte @@ -0,0 +1,471 @@ + + +
    + {#if !$enterpriseLicense} + + {/if} + + + Windmill creates the databases for data tables and Ducklake catalogs on this cluster, and + manages the roles they connect as. The admin login below needs CREATEDB + and CREATEROLE, and is never handed to a job. + + +
    +
    + + + +
    + +
    +
    + Admin password + +
    + +
    + + SSL mode + + {#snippet text()} + Anything below verify-ca sends the managed roles' passwords to whichever server + answers. Windmill trusts the system roots plus the certificate below. + {/snippet} + + + + +
    + +
    + + + + {#if status} +
    + {#if status.configured} + + Configured · {status.database_count} database{status.database_count === 1 ? '' : 's'} + {:else} + + Not configured yet + {/if} +
    + {/if} +
    + + {#if loadError} + {loadError} + {/if} + + {#if status?.last_setup} + {@const report = status.last_setup} +
    +
    + Last setup + {new Date(report.finished_at).toLocaleString()} +
    +
    + {#each report.steps as step} +
    +
    + {#if step.status === 'ok'} + + {:else if step.status === 'warning'} + + {:else} + + {/if} +
    +
    + {step.name} + {step.message} +
    +
    + {/each} +
    +
    + {/if} + + {#if status?.configured} +
    + Databases on the cluster + + Only databases created here can back a data table or a Ducklake catalog, and one in use + cannot be dropped. + + + + + Name + Tag + Used by + + + + + {#each databaseEntries as [name, db]} + + {name} + {db.tag ?? '-'} + + {db.used_by_workspaces?.length ? db.used_by_workspaces.join(', ') : '-'} + + +
    +
    +     +     + {:else} + + + No database yet. Create one below, then pick it in a workspace's data table or + Ducklake settings. + + + {/each} + +     +
    + +
    + dataTable.database.resource_type, (resource_type) => { @@ -451,12 +537,20 @@ } } } + transformInputSelectedText={shortManagedInstanceLabel} id="database-type-select" - class="w-28" + class="w-36" />
    - {#if dataTable.database.resource_type !== 'instance'} + {#if dataTable.database.resource_type === 'external_instance'} + + {:else if dataTable.database.resource_type !== 'instance'} diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index a0d29af434..f6929e0779 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -13,7 +13,7 @@ ducklakes: { name: string catalog: { - resource_type: 'postgresql' | 'mysql' | 'instance' + resource_type: 'postgresql' | 'mysql' | 'instance' | 'external_instance' resource_path?: string // Name of the database when resource_type is instance } storage: { @@ -74,7 +74,6 @@ GitFork, Plus, SettingsIcon, - Wrench } from 'lucide-svelte' import Button from '../common/button/Button.svelte' @@ -91,7 +90,7 @@ import { ScheduleService, SettingService, WorkspaceService } from '$lib/gen' import type { GetSettingsResponse } from '$lib/gen' - import { enterpriseLicense, userWorkspaces, workspaceStore } from '$lib/stores' + import { enterpriseLicense, superadmin, userWorkspaces, workspaceStore } from '$lib/stores' import { base } from '$app/paths' import Toggle from '../Toggle.svelte' import { sendUserToast } from '$lib/toast' @@ -105,9 +104,16 @@ import Popover from '../meltComponents/Popover.svelte' import TextInput from '../text_input/TextInput.svelte' import { slide } from 'svelte/transition' - import { isCustomInstanceDbEnabled, getUnusedInstanceDbName } from './utils.svelte' + import { + isCustomInstanceDbEnabled, + managedInstanceLabels, + shortManagedInstanceLabel, + getUnusedInstanceDbName + } from './utils.svelte' import { resource } from 'runed' + import { isCloudHosted } from '$lib/cloud' import CustomInstanceDbSelect from './CustomInstanceDbSelect.svelte' + import ExternalInstanceDbSelect from './ExternalInstanceDbSelect.svelte' import Label from '../Label.svelte' type Props = { @@ -140,8 +146,14 @@ ducklakeSettings.ducklakes.push({ name, catalog: { - resource_type: $isCustomInstanceDbEnabled ? 'instance' : 'postgresql', - resource_path: $isCustomInstanceDbEnabled ? defaultInstanceDbName() : undefined + // A new entry starts on a kind this instance actually offers, so it is never born + // on one the save would refuse. + resource_type: instanceAvailable + ? 'instance' + : externalInstanceConfigured && $superadmin + ? 'external_instance' + : 'postgresql', + resource_path: instanceAvailable ? defaultInstanceDbName() : undefined }, storage: { storage: undefined, @@ -170,6 +182,77 @@ const customInstanceDbs = resource([() => $workspaceStore], SettingService.listCustomInstanceDbs) + // Superadmin-only endpoints, and the kind is theirs to pick: a workspace admin never loads + // them and sees the option disabled instead of an empty picker. + const externalInstanceStatus = resource([() => $superadmin], ([isSuperadmin]) => + isSuperadmin ? SettingService.getExternalInstancePgStatus() : Promise.resolve(undefined) + ) + const externalInstanceDbs = resource([() => $superadmin], ([isSuperadmin]) => + isSuperadmin ? SettingService.listExternalInstancePgDatabases() : Promise.resolve({}) + ) + let externalInstanceConfigured = $derived(externalInstanceStatus.current?.configured === true) + // Superadmin-only like the ones above, and absent means on. + const instancePgDisabled = resource([() => $superadmin], ([isSuperadmin]) => + isSuperadmin + ? SettingService.getGlobal({ key: 'instance_pg_disabled' }).catch(() => undefined) + : Promise.resolve(undefined) + ) + // Both substrates answer only to a superadmin, so nobody else can be told whether one is on + // offer: they see a managed kind only where an entry already sits on it. + let instancePossible = $derived( + !!$superadmin && !isCloudHosted() && !instancePgDisabled.current + ) + let instanceAvailable = $derived(instancePossible) + + // A kind already saved stays listed whatever the instance offers now. + // Qualified per form, not per row: see DataTableSettings. + let anyInstanceLake = $derived( + ducklakeSettings.ducklakes.some((d) => d.catalog.resource_type === 'instance') + ) + let anyExternalLake = $derived( + ducklakeSettings.ducklakes.some((d) => d.catalog.resource_type === 'external_instance') + ) + function catalogItems(current: string | undefined) { + const showInstance = instancePossible || current === 'instance' + const showExternal = + (externalInstanceConfigured && !!$superadmin) || current === 'external_instance' + const labels = managedInstanceLabels( + instancePossible || anyInstanceLake, + (externalInstanceConfigured && !!$superadmin) || anyExternalLake + ) + const items: { value: string; label: string; disabled?: boolean; subtitle?: string }[] = [ + { value: 'postgresql', label: 'Postgres Resource' }, + { value: 'mysql', label: 'MySQL Resource' } + ] + if (showInstance) { + items.push({ + value: 'instance', + label: labels.instance, + disabled: !instanceAvailable, + subtitle: instanceAvailable + ? undefined + : !$superadmin + ? 'Superadmin only' + : isCloudHosted() + ? 'Not available on cloud' + : "Windmill's database is disabled" + }) + } + if (showExternal) { + items.push({ + value: 'external_instance', + label: labels.external, + disabled: !externalInstanceConfigured || !$superadmin, + subtitle: !$superadmin + ? 'Superadmin only' + : externalInstanceConfigured + ? undefined + : 'No external cluster configured' + }) + } + return items + } + async function onSave() { try { if ( @@ -245,15 +328,13 @@ secondaryStorageNames.refresh() }) - let tableHeadNames = ['Name', 'Catalog', 'Workspace storage', 'Maintenance', '', ''] as const + let tableHeadNames = ['Name', 'Catalog', 'Workspace storage', '', ''] as const let tableHeadTooltips: Partial> = { Name: "Ducklakes are referenced in DuckDB scripts with the ATTACH 'ducklake://name' AS dl; syntax", Catalog: 'Ducklake needs an SQL database to store metadata about the data', 'Workspace storage': - 'Where the data is actually stored, in parquet format. You need to configure a workspace storage first', - Maintenance: - 'Scheduled snapshot expiry, small-file compaction and orphaned-file cleanup, run as jobs on a managed per-lake schedule (EE)' + 'Where the data is actually stored, in parquet format. You need to configure a workspace storage first' } let confirmationModal = createAsyncConfirmationModal() @@ -283,10 +364,9 @@ This workspace is a fork, and these settings are its own copy. Lakes marked isolated read the parent's tables through defer views and write to a fork-scoped namespace that is cleaned up when the fork is deleted. Lakes marked - shared with parent read and write the parent's physical - lake directly — editing their catalog or storage here repoints the shared lake for this - fork's jobs. The choice is made per lake when the fork is created and cannot be changed - here. + shared with parent read and write the parent's physical lake + directly — editing their catalog or storage here repoints the shared lake for this fork's jobs. + The choice is made per lake when the fork is created and cannot be changed here.
    {/if} @@ -359,8 +439,8 @@ isolated - Writes go to a fork-scoped namespace; reads of tables not yet materialized in - this fork defer to the parent. Deleting the fork cleans the namespace up. + Writes go to a fork-scoped namespace; reads of tables not yet materialized in this + fork defer to the parent. Deleting the fork cleans the namespace up. {/if}
    @@ -373,17 +453,13 @@ Use Windmill's PostgreSQL instance as a catalog + {:else if ducklake.catalog.resource_type === 'external_instance'} + + Use a database Windmill manages on the external PostgreSQL cluster as a catalog + {/if} (value = i)} + placeholder="Search or create..." + showPlaceholderOnOpen + {items} + id="external-instance-db-select" + disabled={!$superadmin} + noItemsMsg="Start typing to create a new database" + > + {#snippet endSnippet({ item })} + {@render sharedWorkspacesWarning(item.value)} + {/snippet} + + {#if value} +
    + {#if unknownName} + + {:else if $superadmin} + {@render sharedWorkspacesWarning(value)} +
    + {/if} +
    + {/if} +
    + +{#snippet sharedWorkspacesWarning(dbname: string)} + {@const others = otherWorkspaces(dbname)} + {#if others.length > 0} + + + {#snippet text()} + This database is also used by workspace{others.length > 1 ? 's' : ''} + {others.join(', ')}. Any data written here will be shared + with {others.length > 1 ? 'them' : 'it'}. + {/snippet} + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte b/frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte index 9932495fce..6072cdad3f 100644 --- a/frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte +++ b/frontend/src/lib/components/workspaceSettings/InstanceRolesButton.svelte @@ -2,14 +2,18 @@ import { Badge, Button, Drawer, DrawerContent } from '../common' import { Users } from 'lucide-svelte' import DataTableRolesSection from './DataTableRolesSection.svelte' + import type { DatatableRoleCluster } from '$lib/gen' let { hideTrigger = false, + cluster, onChanged, unavailable }: { /** Mount the drawer without its button, for a caller that opens it with `open()`. */ hideTrigger?: boolean + /** Whose catalog to manage. A role is a login on one cluster. */ + cluster?: DatatableRoleCluster /** Called after every change to the instance roles. */ onChanged?: () => void /** Why the roles cannot be managed here: the button stays, disabled with this reason, so the @@ -49,13 +53,13 @@ drawer?.closeDrawer()} - tooltip="A data table role is a real Postgres login on this instance, shared by every instance database. A job that names one connects as it, and Postgres decides what it may touch. Which people may use a role on a given data table, and what it may do there, is set per data table, in its roles drawer." + tooltip="A data table role is a real Postgres login on the cluster it belongs to, shared by every database Windmill manages there. A job that names one connects as it, and Postgres decides what it may touch. Which people may use a role on a given data table, and what it may do there, is set per data table, in its roles drawer." > {#snippet titleExtra()} Beta {/snippet} {#key openCount} - + {/key} diff --git a/frontend/src/lib/components/workspaceSettings/addDataTableModel.test.ts b/frontend/src/lib/components/workspaceSettings/addDataTableModel.test.ts index cb42d73075..9461a7b04b 100644 --- a/frontend/src/lib/components/workspaceSettings/addDataTableModel.test.ts +++ b/frontend/src/lib/components/workspaceSettings/addDataTableModel.test.ts @@ -23,6 +23,7 @@ const getSettingsMock = vi.fn() const editDataTableConfigMock = vi.fn() const testDataTableConnectionMock = vi.fn() const setupCustomInstanceDbMock = vi.fn() +const createExternalInstanceDbMock = vi.fn() vi.mock('$lib/gen', () => ({ VariableService: { existsVariable: (...a: any[]) => existsVariableMock(...a), @@ -36,7 +37,10 @@ vi.mock('$lib/gen', () => ({ createResource: vi.fn(), updateResource: vi.fn() }, - SettingService: { setupCustomInstanceDb: (...a: any[]) => setupCustomInstanceDbMock(...a) }, + SettingService: { + setupCustomInstanceDb: (...a: any[]) => setupCustomInstanceDbMock(...a), + createExternalInstancePgDatabase: (...a: any[]) => createExternalInstanceDbMock(...a) + }, WorkspaceService: { getSettings: (...a: any[]) => getSettingsMock(...a), editDataTableConfig: (...a: any[]) => editDataTableConfigMock(...a), @@ -239,6 +243,48 @@ describe('runSetup rolling the instance row back', () => { }) }) +// A database on the external cluster belongs to whoever created it, and the endpoint refuses a +// name it already holds. Skipping the create on any registered name is how a run ends up +// attached to somebody else's data without the sharing warning the existing-database branch +// shows -- so only the databases this run made may make a retry skip it. +describe('runSetup creating a database on the external cluster', () => { + function creatingExternal(): WizardState { + const state = newWizardState({ name: 'main', projectName: 'x', folder: 'f/team' }) + state.provider = 'external_instance' + state.external = { mode: 'create', dbName: 'dt_new' } + return state + } + const externalDeps = (createdExternalDbs: string[] = []) => + ({ + workspace: 'w', + onProgress: () => {}, + claims: noClaims, + username: 'alice', + createdProjects: [], + createdExternalDbs + }) as any + + beforeEach(() => { + vi.clearAllMocks() + getSettingsMock.mockResolvedValue({ datatable: { datatables: {} } }) + editDataTableConfigMock.mockResolvedValue(undefined) + testDataTableConnectionMock.mockResolvedValue({ can_create_table: true }) + }) + + it('creates the database on a first attempt', async () => { + const result = await runSetup(creatingExternal(), externalDeps()) + expect(result.ok).toBe(true) + expect(createExternalInstanceDbMock).toHaveBeenCalledTimes(1) + expect(result.createdExternalDbs).toContain('dt_new') + }) + + it('skips the create only for a database it made itself', async () => { + const result = await runSetup(creatingExternal(), externalDeps(['dt_new'])) + expect(result.ok).toBe(true) + expect(createExternalInstanceDbMock).not.toHaveBeenCalled() + }) +}) + // The fields are the connection; a connection string is a way of writing one down. Reading the // resource back out of the string is what let a URI grammar gap change what got saved. describe('newResourceParts', () => { diff --git a/frontend/src/lib/components/workspaceSettings/addDataTableModel.ts b/frontend/src/lib/components/workspaceSettings/addDataTableModel.ts index 2625786991..073d496d0c 100644 --- a/frontend/src/lib/components/workspaceSettings/addDataTableModel.ts +++ b/frontend/src/lib/components/workspaceSettings/addDataTableModel.ts @@ -42,7 +42,7 @@ import { type SupabaseProject } from './supabaseProvisioning' -export type Provider = 'supabase' | 'instance' | 'resource' +export type Provider = 'supabase' | 'instance' | 'external_instance' | 'resource' export type WizardState = { step: 1 | 2 | 3 @@ -61,6 +61,8 @@ export type WizardState = { connectionMode: SupabaseConnectionMode } instance: { mode: 'existing' | 'create'; dbName: string | undefined } + /** Same shape as `instance`, on the Postgres cluster Windmill administers elsewhere. */ + external: { mode: 'existing' | 'create'; dbName: string | undefined } /** * One list: the workspace's Postgres resources, plus the one about to exist. A * connection string is not an alternative to a resource, it is how one is written -- @@ -107,6 +109,7 @@ export function newWizardState(defaults: { connectionMode: 'session' }, instance: { mode: 'create', dbName: undefined }, + external: { mode: 'create', dbName: undefined }, own: { resourcePath: undefined, creating: false, @@ -137,6 +140,7 @@ export function intentComplete(state: WizardState): boolean { : !!state.supabase.project && !!state.supabase.password } if (state.provider === 'instance') return !!state.instance.dbName?.trim() + if (state.provider === 'external_instance') return !!state.external.dbName?.trim() if (!state.own.creating) return !!state.own.resourcePath // Text that will not parse leaves the fields on their last good values, which is what makes // it correctable -- but the connection on screen is then not the one they describe, and @@ -307,6 +311,7 @@ export type RunStepKey = | 'create_project' | 'wait_healthy' | 'save_credentials' + | 'create_external' | 'setup_instance' | 'check' @@ -331,6 +336,13 @@ export function plan(state: WizardState): { key: RunStepKey; title: string }[] { key: 'setup_instance', title: `Setting up ${state.instance.dbName} in the Windmill database` }) + } else if (state.provider === 'external_instance') { + if (state.external.mode === 'create') { + steps.push({ + key: 'create_external', + title: `Creating ${state.external.dbName} on the external cluster` + }) + } } else if (state.own.creating) { steps.push({ key: 'save_credentials', title: `Saving the connection to ${path}` }) } @@ -362,6 +374,16 @@ export type RunDeps = { * is really there, since the name is also recorded when a create could not be confirmed. */ createdProjects: CreatedProject[] + /** + * The databases earlier attempts in this session created on the external cluster. Only these + * make a retry skip the create: any other registered name is somebody else's database, and + * attaching to it silently would share their data without the warning the existing-database + * branch shows. + */ + createdExternalDbs?: string[] + /** Called once a database lands on the external cluster, so the caller's registry — which + * names the next run's default and validates it — is not a page-load-old view. */ + onExternalDbsChanged?: () => Promise /** * What earlier attempts in this session wrote, and this one may therefore write over again. * The pre-flight checks the names are free, but the Supabase branch then spends minutes @@ -391,6 +413,8 @@ export type RunResult = { * later attempt may write over it. */ createdProjects: CreatedProject[] + /** The external-cluster databases this session created, kept for the same reason. */ + createdExternalDbs: string[] /** What this run holds now, for the next attempt to be given back. */ claims: Claims } @@ -410,6 +434,17 @@ async function exists(kind: 'variable' | 'resource', workspace: string, path: st : ResourceService.existsResource({ workspace, path }) } +/** + * What identifies the database a row points at. The kind belongs in it: `instance` and + * `external_instance` are different databases that may carry the same name, so a row repointed + * from one to the other while this run probes must not read as the row this run wrote. + */ +function rowMark(database: { resource_type?: string; resource_path?: string } | undefined) { + return database?.resource_path === undefined + ? undefined + : `${database.resource_type ?? ''}:${database.resource_path}` +} + /** * Adds the data table to the workspace config, once everything it points at exists. * `edit_datatable_config` replaces the whole map, so the rest is read back and sent with @@ -419,16 +454,16 @@ async function writeRow( deps: RunDeps, claims: Claims, name: string, - database: { resource_type: 'postgresql' | 'instance'; resource_path: string } + database: { + resource_type: 'postgresql' | 'instance' | 'external_instance' + resource_path: string + } ): Promise { const settings = await WorkspaceService.getSettings({ workspace: deps.workspace }) const datatables: Record = { ...(settings.datatable?.datatables ?? {}) } // Free when the pre-flight looked, taken by the time we write: repointing it here would // silently hand another admin's data table a database they never chose. - if ( - datatables[name] && - !stillOurs(claims, 'row', name, datatables[name]?.database?.resource_path) - ) { + if (datatables[name] && !stillOurs(claims, 'row', name, rowMark(datatables[name]?.database))) { throw new Error( `A data table called ${name} was created while this setup was running. Choose another name and try again.` ) @@ -438,7 +473,7 @@ async function writeRow( workspace: deps.workspace, requestBody: { settings: { datatables }, renames: [], deleted_datatables: [] } }) - return claim(claims, 'row', name, database.resource_path) + return claim(claims, 'row', name, rowMark(database)!) } /** @@ -455,7 +490,7 @@ async function removeRow(deps: RunDeps, claims: Claims, name: string): Promise { if (!createdProjects.some((p) => p.path === at)) @@ -595,7 +631,8 @@ export async function runSetup(state: WizardState, deps: RunDeps): Promise p.path === path) const instanceName = state.instance.dbName?.trim() ?? '' + const externalName = state.external.dbName?.trim() ?? '' let project = state.supabase.project let resourcePath = @@ -740,6 +778,18 @@ export async function runSetup(state: WizardState, deps: RunDeps): Promise Date: Tue, 29 Sep 2026 14:46:51 +0200 Subject: [PATCH 70/91] fix: refuse flow preview restarts from runs the caller cannot read (#11407) --- .../tests/preview_restart_source_access.rs | 77 +++++++++++++++++++ backend/windmill-api/src/jobs.rs | 7 ++ 2 files changed, 84 insertions(+) create mode 100644 backend/tests/preview_restart_source_access.rs diff --git a/backend/tests/preview_restart_source_access.rs b/backend/tests/preview_restart_source_access.rs new file mode 100644 index 0000000000..b6a552af3d --- /dev/null +++ b/backend/tests/preview_restart_source_access.rs @@ -0,0 +1,77 @@ +//! A flow preview restarted from an earlier run copies that run's step results, so it must be +//! refused unless the caller can see every run the request names. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::JobPayload; +use windmill_test_utils::*; + +fn identity_flow() -> serde_json::Value { + json!({ "modules": [ + { "id": "a", "value": { "type": "identity" } }, + { "id": "b", "value": { "type": "identity" } }, + ]}) +} + +async fn completed_run(db: &Pool, port: u16, username: &str, email: &str) -> String { + RunJob::from(JobPayload::RawFlow { + value: serde_json::from_value(identity_flow()).unwrap(), + path: None, + restarted_from: None, + }) + .as_user(username, email) + .run_until_complete(db, false, port) + .await + .id + .to_string() +} + +#[sqlx::test(fixtures("base"))] +async fn test_preview_restart_requires_source_read_access(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let admin_run = completed_run(&db, port, "test-user", "test@windmill.dev").await; + let own_run = completed_run(&db, port, "test-user-2", "test2@windmill.dev").await; + + let restart = |restarted_from: serde_json::Value| { + reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/preview_flow" + )) + .bearer_auth("SECRET_TOKEN_2") + .json( + &json!({ "args": {}, "value": identity_flow(), "restarted_from": restarted_from }), + ) + .send() + }; + + let resp = restart(json!({ "flow_job_id": admin_run, "step_id": "b" })).await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + + let resp = restart(json!({ + "flow_job_id": own_run, "step_id": "b", + "nested": { "flow_job_id": admin_run, "step_id": "b" }, + })) + .await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + + let resp = restart(json!({ "flow_job_id": own_run, "step_id": "b" })).await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + + // A run hidden on its own but inside a flow the caller can read is readable, as on the run page. + let child_run = completed_run(&db, port, "test-user", "test@windmill.dev").await; + sqlx::query("UPDATE v2_job SET parent_job = $1::uuid WHERE id = $2::uuid") + .bind(&own_run) + .bind(&child_run) + .execute(&db) + .await?; + let resp = restart(json!({ + "flow_job_id": own_run, "step_id": "b", + "nested": { "flow_job_id": child_run, "step_id": "b" }, + })) + .await?; + assert_eq!(resp.status(), 201, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 27e34b6ef2..10631dbe2b 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -9655,6 +9655,13 @@ async fn run_preview_flow_job( // jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script. check_scopes(&authed, || format!("jobs:run"))?; require_path_read_access_for_preview(&authed, &raw_flow.path)?; + // Restarting copies the source runs' step results into the new run, and the queue resolves + // them with the service pool, so every run the request names must be readable as the caller. + let mut level = raw_flow.restarted_from.as_ref(); + while let Some(r) = level { + require_job_update_read_access(&db, &user_db, &authed, &w_id, &r.flow_job_id, None).await?; + level = r.nested.as_deref(); + } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(raw_flow.tag.clone()); let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); From d44c901647422345f869a9e953431504c5065d61 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 29 Sep 2026 14:47:22 +0200 Subject: [PATCH 71/91] replace the AI sessions beta banner with a feedback link (#11401) * feat: make the AI sessions beta banner dismissible Co-Authored-By: Claude Opus 5.5 (1M context) * feat: move AI sessions feedback link into assistant settings Co-Authored-By: Claude Opus 5.5 (1M context) * refactor: reduce the sessions beta gate setter to opt-in only Co-Authored-By: Claude Opus 5.5 (1M context) * fix: size the sessions activate buttons with unifiedSize Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep the sessions page activate button at its previous height Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../copilot/chat/AiChatLayout.svelte | 2 +- .../chat/AssistantSettingsModal.svelte | 27 +++++++- .../components/copilot/chat/global/gate.ts | 26 +++---- .../components/sessions/SessionWrapper.svelte | 2 - .../sessions/SessionsBetaBanner.svelte | 67 +++++-------------- .../src/routes/(root)/(logged)/+layout.svelte | 4 +- .../(root)/(logged)/sessions/+page.svelte | 4 +- 7 files changed, 55 insertions(+), 77 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte b/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte index 5767085c92..1a1412b9f8 100644 --- a/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte +++ b/frontend/src/lib/components/copilot/chat/AiChatLayout.svelte @@ -132,7 +132,7 @@ {/await}
    {#if showSessionsBetaBanner} - + {/if} {/if} diff --git a/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte b/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte index 8e299a4d3e..6c6eaf5247 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte @@ -14,7 +14,15 @@ callers that already know which section they mean, such as the "+" menu's Manage - -
    + +
    - {#if variant === 'session'} - AI Sessions is in beta - · - - · - - {:else} - Try AI Sessions (beta) - · - - {/if} + Try AI Sessions (beta) + · +
    diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index ca2b439025..5cf6e4f5fa 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -275,8 +275,8 @@ // nest the whole experience. Hide it when embedded. const embedded = BROWSER && window.self !== window.top - // AI sessions (beta) are on unless the user opted out from the banner under - // the session chat. The Workspace ⇄ Sessions switch is the only entry point, + // AI sessions (beta) are on unless this browser opted out earlier (see + // `global/gate.ts`). The Workspace ⇄ Sessions switch is the only entry point, // so it follows the gate; opted-out users get the legacy Ask-AI pane instead. // The /sessions page has its own gate for direct navigation. const globalAiEnabled = isGlobalAiEnabled() diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 689afdbb9f..577f7132ce 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -50,7 +50,7 @@ import { markSessionRecovered } from '$lib/components/sessions/sessionRecoveryNotice.svelte' import { isGlobalAiEnabled, - setSessionsBetaOptOut + clearSessionsBetaOptOut } from '$lib/components/copilot/chat/global/gate' import { setToolCompletionListener } from '$lib/components/copilot/chat/shared' import { registerToolDisplayActionHandler } from '$lib/components/copilot/chat/createdResourceActions.svelte' @@ -908,7 +908,7 @@

    AI Sessions are deactivated

    You switched back to the legacy chat. Activate AI Sessions (beta) to open this page.

    -
    From ede4103b00c6ada3b5ea8ba3e98bf6b78a791d63 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 29 Sep 2026 14:47:42 +0200 Subject: [PATCH 72/91] feat: share AI guidance with the CLI skills and lint flow groups (#11397) * feat: lint AI agent tool names and flow groups in wmill lint * refactor: assemble chat and CLI AI guidance from one topic table * feat: share flow groups, reuse and pipeline guidance with the CLI skills * feat: share raw app, data table and secret guidance between chat and CLI * docs: document the shared AI guidance source for contributors * fix: keep wmill lint running on flows with malformed collections * fix: reject skill descriptions that are not plain YAML text * fix: tighten fence typos, script base scope and app prompt order * fix: skip tool name checks on agent steps linked to a saved agent * fix: catch any misspelled prompt fence and soften the tool name claim * fix: align cli eval harness with the files and steps wmill init adds * fix: drop cli eval checks that expect unrequested deploy commands * fix: list ansible as mainless and c# Main in script base guidance --- .agents/skills/ai-chat/SKILL.md | 9 + AGENTS.md | 1 + ai_evals/cases/cli.yaml | 24 - ai_evals/modes/cli.ts | 6 +- cli/src/commands/app/generate_agents.ts | 9 +- cli/src/commands/lint/flow_semantics.ts | 312 ++++++++ cli/src/commands/lint/lint.ts | 13 +- cli/src/commands/sync/sync.ts | 20 +- cli/src/guidance/core.ts | 7 +- cli/src/guidance/skills.gen.ts | 719 +++++++++++++++++- cli/test/lint_command_unit.test.ts | 164 ++++ .../lib/components/copilot/chat/app/core.ts | 78 +- .../lib/components/copilot/chat/flow/core.ts | 16 +- .../copilot/chat/flow/openFlow.json | 2 +- .../components/copilot/chat/global/core.ts | 22 +- openflow.openapi.yaml | 6 +- system_prompts/README.md | 148 ++-- .../auto-generated/cli/cli-commands.md | 2 +- system_prompts/auto-generated/flow.md | 44 +- system_prompts/auto-generated/index.ts | 8 +- system_prompts/auto-generated/prompts.ts | 181 +++-- system_prompts/auto-generated/script.md | 45 +- .../skills/cli-commands/SKILL.md | 2 +- .../auto-generated/skills/raw-app/SKILL.md | 44 +- .../auto-generated/skills/resources/SKILL.md | 8 + .../auto-generated/skills/write-flow/SKILL.md | 48 +- .../skills/write-pipeline/SKILL.md | 125 +++ .../skills/write-script-ansible/SKILL.md | 23 + .../skills/write-script-bash/SKILL.md | 23 + .../skills/write-script-bigquery/SKILL.md | 23 + .../skills/write-script-bun/SKILL.md | 30 +- .../skills/write-script-bunnative/SKILL.md | 30 +- .../skills/write-script-csharp/SKILL.md | 23 + .../skills/write-script-deno/SKILL.md | 30 +- .../skills/write-script-duckdb/SKILL.md | 23 + .../skills/write-script-go/SKILL.md | 23 + .../skills/write-script-graphql/SKILL.md | 23 + .../skills/write-script-java/SKILL.md | 23 + .../skills/write-script-mssql/SKILL.md | 23 + .../skills/write-script-mysql/SKILL.md | 23 + .../skills/write-script-php/SKILL.md | 23 + .../skills/write-script-postgresql/SKILL.md | 23 + .../skills/write-script-powershell/SKILL.md | 23 + .../skills/write-script-python3/SKILL.md | 27 +- .../skills/write-script-rlang/SKILL.md | 23 + .../skills/write-script-rust/SKILL.md | 23 + .../skills/write-script-snowflake/SKILL.md | 23 + .../skills/write-workflow-as-code/SKILL.md | 4 +- system_prompts/base/flow-base.md | 58 +- system_prompts/base/pipeline-base.md | 38 +- system_prompts/base/raw-app-cli.md | 7 +- system_prompts/base/raw-app.md | 44 +- system_prompts/base/resources.md | 14 + system_prompts/base/script-base.md | 19 +- system_prompts/base/script-cli.md | 44 ++ system_prompts/base/workflow-as-code.md | 3 + system_prompts/generate.py | 481 ++++++------ system_prompts/languages/bun.md | 8 +- system_prompts/languages/bunnative.md | 8 +- system_prompts/languages/deno.md | 8 +- system_prompts/languages/python3.md | 2 +- system_prompts/utils.py | 53 ++ 62 files changed, 2681 insertions(+), 658 deletions(-) create mode 100644 cli/src/commands/lint/flow_semantics.ts create mode 100644 system_prompts/auto-generated/skills/write-pipeline/SKILL.md create mode 100644 system_prompts/base/script-cli.md diff --git a/.agents/skills/ai-chat/SKILL.md b/.agents/skills/ai-chat/SKILL.md index f670b2ffb7..490c459db9 100644 --- a/.agents/skills/ai-chat/SKILL.md +++ b/.agents/skills/ai-chat/SKILL.md @@ -11,6 +11,15 @@ authoring and the full run reference. Run the affected mode **before** your change and **after**, same model(s), same cases. +## Where guidance goes + +What the model should know about Windmill itself (flow shapes, groups, data tables, app +access, secrets…) belongs in `system_prompts/base/` or `system_prompts/languages/`, which also +feed the CLI's skills — not in a `*/core.ts` prompt builder, where only this chat would see it. +The builders add tool plumbing and runtime values only. Scope a sentence to one consumer with +`` / ``, keep chat tool names out of the shared files, and +rerun `python system_prompts/generate.py`. `system_prompts/README.md` has the mechanics. + ## Measure the window first, and cumulative second Optimize **`finalContextTokens`** (window occupancy — what drives overflow and diff --git a/AGENTS.md b/AGENTS.md index 850538f080..f4bbfec6ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,7 @@ Open-source platform for internal tools, workflows, API integrations, background - **Brand/UI guidelines**: `frontend/brand-guidelines.md` - **Domain vocabulary**: `CONTEXT.md` — the words this codebase uses for its own concepts (step, step setting, trigger step, …). Name things the way it does. - **CLI commands**: when adding/modifying/removing a command, subcommand, option, or description in `cli/src/commands/`, run `python system_prompts/generate.py` to refresh `system_prompts/auto-generated/` and `cli/src/guidance/skills.gen.ts`. The CLI docs the agents use to operate `wmill` are derived from the source — stale generated files give agents the wrong flags. +- **AI guidance** (chat prompts and CLI skills): what the AI knows about Windmill itself — how flows, scripts, apps, pipelines and resources work — is written once, in `system_prompts/base/` and `system_prompts/languages/`, and `system_prompts/generate.py` builds both the chat's `$system_prompts` exports and the CLI's skills from it. Add or change such guidance there, never inline in one consumer: a sentence only one side should see goes in a `` / `` block, and a new file reaches both through the `TOPICS` table. The chat prompt builders under `frontend/src/lib/components/copilot/chat/` add only tool plumbing and runtime values, and shared markdown names no chat tool, since it reaches every chat mode. Rerun `generate.py` after editing; `system_prompts/README.md` has the details. - **Session recorder**: `frontend/src/lib/components/recording/` is also the recorder `wmill app dev --recording` serves, vendored into the CLI as `cli/src/commands/app/devRecorderBundle.gen.ts`. After changing `rawAppSnapshot.ts` or `rawAppRecording.svelte.ts`, run `bun run gen:dev-recorder` from `cli/` (`cli/test/dev_recorder_bundle_unit.test.ts` fails otherwise). - **Raw-app policy**: `frontend/src/lib/components/raw_apps/rawAppPolicy.ts` also derives the policy the server's raw-app deploy stores, vendored into the bundle job as `backend/windmill-api/src/apps_raw_policy.gen.js`. After changing it or anything it imports, run `bun run gen:app-policy` from `cli/` (`cli/test/app_policy_bundle_unit.test.ts` fails otherwise). It rides in the job rather than being read from the CLI the job runs because the images install `windmill-cli` unpinned, so an image can carry one older than its server. diff --git a/ai_evals/cases/cli.yaml b/ai_evals/cases/cli.yaml index d57425e827..b7abcf7f14 100644 --- a/ai_evals/cases/cli.yaml +++ b/ai_evals/cases/cli.yaml @@ -11,12 +11,6 @@ forbiddenSkills: - write-script-python3 - write-flow - orderedAssistantMentions: - - wmill generate-metadata - - wmill sync push - orderedProposedCommands: - - wmill generate-metadata - - wmill sync push forbiddenExecutedCommands: - ^wmill generate-metadata - ^wmill sync push @@ -38,12 +32,6 @@ - write-flow forbiddenSkills: - write-script-python3 - orderedAssistantMentions: - - wmill generate-metadata - - wmill sync push - orderedProposedCommands: - - wmill generate-metadata - - wmill sync push forbiddenExecutedCommands: - ^wmill generate-metadata - ^wmill sync push @@ -65,12 +53,6 @@ forbiddenSkills: - write-script-bun - write-flow - orderedAssistantMentions: - - wmill generate-metadata - - wmill sync push - orderedProposedCommands: - - wmill generate-metadata - - wmill sync push forbiddenExecutedCommands: - ^wmill generate-metadata - ^wmill sync push @@ -114,12 +96,6 @@ - write-flow requiredSkillsBeforeFirstMutation: - write-flow - orderedAssistantMentions: - - wmill generate-metadata - - wmill sync push - orderedProposedCommands: - - wmill generate-metadata - - wmill sync push forbiddenExecutedCommands: - ^wmill generate-metadata - ^wmill sync push diff --git a/ai_evals/modes/cli.ts b/ai_evals/modes/cli.ts index 3ce7cf1c16..205cdfa9c4 100644 --- a/ai_evals/modes/cli.ts +++ b/ai_evals/modes/cli.ts @@ -17,8 +17,12 @@ import type { BenchmarkArtifactFile, CliTrace, ModeRunner } from "../core/types" const IGNORE_WORKSPACE_FILES = new Set([ ".claude", + ".agents", "AGENTS.md", + "AGENTS.wmill.md", "CLAUDE.md", + // the guidance has the agent create one for every new folder + "folder.meta.yaml", "rt.d.ts", ".wmill-benchmark-bin", ".wmill-benchmark-wmill-invocations.log", @@ -89,7 +93,7 @@ export function createCliModeRunner( const run = await runPromptAndCapture( renderedPrompt, workspaceDir, - context.evalCase?.runtime?.maxTurns ?? 6, + context.evalCase?.runtime?.maxTurns ?? 12, modelConfig ); const workspaceFiles = await readDirectoryFiles(workspaceDir, { ignore: IGNORE_WORKSPACE_FILES }); diff --git a/cli/src/commands/app/generate_agents.ts b/cli/src/commands/app/generate_agents.ts index eec86a8b19..6e3bac6888 100644 --- a/cli/src/commands/app/generate_agents.ts +++ b/cli/src/commands/app/generate_agents.ts @@ -186,7 +186,14 @@ export async function regenerateAgentDocs( } // Read local data configuration from raw_app.yaml - let localData: { tables?: string[]; datatable?: string; schema?: string } | undefined; + let localData: + | { + tables?: string[]; + datatable?: string; + schema?: string; + roles?: Record; + } + | undefined; try { const rawApp = (await yamlParseFile(rawAppPath)) as Record; if (rawApp.data && typeof rawApp.data === "object") { diff --git a/cli/src/commands/lint/flow_semantics.ts b/cli/src/commands/lint/flow_semantics.ts new file mode 100644 index 0000000000..26b72a8867 --- /dev/null +++ b/cli/src/commands/lint/flow_semantics.ts @@ -0,0 +1,312 @@ +/** + * Flow checks the OpenFlow JSON schema cannot express. Errors are what fails + * a run of the flow or stops the flow editor from drawing it; warnings are + * what the editor tolerates but renders wrong. + */ + +// `TOOL_NAME_REGEX` and `flow_module_tool_name` in +// backend/windmill-worker/src/ai_executor.rs; `getToolNameError` in +// frontend/src/lib/components/flows/agentToolUtils.ts applies the same rules. +const TOOL_NAME_REGEX = /^[a-zA-Z0-9_]+$/; +const WEBSEARCH_ENABLED_NAME = "__wm_web_search"; + +// `forbiddenIds` in frontend/src/lib/components/flows/idUtils.ts. +const RESERVED_IDS = new Set([ + "do", + "bg", + "ctx", + "state", + "if", + "else", + "for", + "delete", + "while", + "new", + "in", + "failure", + "preprocessor", + "__wm_agent_root", + "as", + "Input", + "Result", + "Trigger", +]); + +// `NoteColor` in frontend/src/lib/components/graph/noteColors.ts. +const PALETTE = [ + "yellow", + "blue", + "green", + "purple", + "pink", + "orange", + "red", + "cyan", + "lime", + "gray", +]; + +// Nodes the flow graph synthesizes (`VIRTUAL_NODE_IDS` in +// frontend/src/lib/components/graph/groupDetectionUtils.ts). +const VIRTUAL_NODE_IDS = new Set(["Input", "Result", "Trigger"]); + +export interface FlowSemanticReport { + errors: string[]; + warnings: string[]; +} + +interface ModuleList { + pointer: string; + modules: any[]; +} + +// These checks also run on a flow that failed schema validation, so any +// collection may have the wrong type; treating it as empty leaves reporting it +// to the schema errors instead of aborting the whole lint run. +function asArray(value: unknown): any[] { + return Array.isArray(value) ? value : []; +} + +/** + * Every module list a group can span, and every aiagent module, anywhere in the + * tree. The lists match `getContainerInnerArrays` in + * frontend/src/lib/components/graph/groupEditor.svelte.ts: agent tools are not + * steps of the graph, so no group can reach them. + */ +function walkModules( + modules: unknown, + pointer: string, + lists: ModuleList[], + agents: { pointer: string; module: any }[], +) { + if (!Array.isArray(modules)) return; + lists.push({ pointer, modules }); + modules.forEach((m, i) => { + const value = m?.value; + const at = `${pointer}/${i}/value`; + switch (value?.type) { + case "forloopflow": + case "whileloopflow": + walkModules(value.modules, `${at}/modules`, lists, agents); + break; + case "branchone": + walkModules(value.default, `${at}/default`, lists, agents); + asArray(value.branches).forEach((b: any, bi: number) => + walkModules(b?.modules, `${at}/branches/${bi}/modules`, lists, agents) + ); + break; + case "branchall": + asArray(value.branches).forEach((b: any, bi: number) => + walkModules(b?.modules, `${at}/branches/${bi}/modules`, lists, agents) + ); + break; + case "aiagent": + collectAgents(m, `${pointer}/${i}`, agents); + break; + } + }); +} + +/** An aiagent module plus any agent that is itself one of its tools. */ +function collectAgents( + module: any, + pointer: string, + agents: { pointer: string; module: any }[], +) { + // A step linked to an `ai_agent` resource runs the resource's tools and ignores + // its own (`ai_executor.rs`), so those are left unchecked. + if (typeof module?.value?.agent === "string" && module.value.agent !== "") { + return; + } + agents.push({ pointer, module }); + asArray(module?.value?.tools).forEach((tool: any, i: number) => { + if (tool?.value?.type === "aiagent") { + collectAgents(tool, `${pointer}/value/tools/${i}`, agents); + } + }); +} + +function checkAgentTools( + agent: { pointer: string; module: any }, + report: FlowSemanticReport, +) { + const tools = agent.module?.value?.tools; + if (!Array.isArray(tools)) return; + const seen = new Map(); + tools.forEach((tool: any, i: number) => { + // An mcp entry exposes its server's own tool names and a websearch entry is + // named by `WEBSEARCH_ENABLED_NAME`; neither summary reaches the model. + const kind = tool?.value?.tool_type; + if (kind === "mcp" || kind === "websearch") return; + const at = `${agent.pointer}/value/tools/${i}`; + const name = tool?.summary; + if (typeof name !== "string" || name.length === 0) { + report.errors.push( + `${at} has no summary: a tool's summary is the name the agent calls it by`, + ); + return; + } + if (!TOOL_NAME_REGEX.test(name)) { + report.errors.push( + `${at} tool name '${name}' may only contain letters, numbers and underscores (a run that offers this tool fails with 'Invalid tool name')`, + ); + return; + } + if (name === WEBSEARCH_ENABLED_NAME) { + report.errors.push( + `${at} tool name '${name}' is reserved for enabling web search`, + ); + return; + } + if (seen.has(name)) { + report.errors.push( + `${at} tool name '${name}' is already used by tool ${seen.get(name)} of this agent`, + ); + return; + } + seen.set(name, i); + if (RESERVED_IDS.has(name)) { + report.warnings.push( + `${at} tool name '${name}' is a reserved id; the flow editor refuses it`, + ); + } + }); +} + +function checkColor( + at: string, + color: unknown, + report: FlowSemanticReport, +) { + if (color === undefined || color === null) return; + if (typeof color !== "string" || !PALETTE.includes(color)) { + report.warnings.push( + `${at} color '${color}' renders unstyled; use one of: ${PALETTE.join(", ")}`, + ); + } +} + +/** + * Mirrors `buildStructureTree` in frontend/src/lib/components/graph/flowStructure.ts, + * whose throws make the editor draw no step at all. + */ +function checkGroups( + groups: unknown, + lists: ModuleList[], + report: FlowSemanticReport, +) { + if (!Array.isArray(groups)) return; + const location = new Map(); + lists.forEach((l, list) => + l.modules.forEach((m, index) => { + if (typeof m?.id === "string") location.set(m.id, { list, index }); + }) + ); + + const placed: { at: string; list: number; start: number; end: number }[] = + []; + const keys = new Set(); + groups.forEach((g: any, i: number) => { + const at = `/value/groups/${i}`; + checkColor(at, g?.color, report); + const startId = g?.start_id; + const endId = g?.end_id; + if (typeof startId !== "string" || typeof endId !== "string") return; + + const key = `${startId}:${endId}`; + if (keys.has(key)) { + report.errors.push( + `${at} spans the same steps as another group (${startId} to ${endId})`, + ); + return; + } + keys.add(key); + + for (const id of [startId, endId]) { + if (VIRTUAL_NODE_IDS.has(id)) { + report.errors.push(`${at} cannot include the '${id}' node`); + return; + } + if (!location.has(id)) { + report.errors.push( + id === "preprocessor" || id === "failure" + ? `${at} cannot include the '${id}' module` + : `${at} references '${id}', which is not a step of this flow`, + ); + return; + } + } + const start = location.get(startId)!; + const end = location.get(endId)!; + if (start.list !== end.list) { + report.errors.push( + `${at} starts at '${startId}' and ends at '${endId}', which are not in the same branch`, + ); + return; + } + if (start.index > end.index) { + report.errors.push( + `${at} starts at '${startId}', which comes after its end '${endId}'`, + ); + return; + } + placed.push({ at, list: start.list, start: start.index, end: end.index }); + }); + + for (let a = 0; a < placed.length; a++) { + for (let b = a + 1; b < placed.length; b++) { + const x = placed[a]; + const y = placed[b]; + if (x.list !== y.list) continue; + const disjoint = x.end < y.start || y.end < x.start; + const nested = (x.start <= y.start && y.end <= x.end) || + (y.start <= x.start && x.end <= y.end); + if (!disjoint && !nested) { + report.errors.push( + `${y.at} partly overlaps ${x.at}: groups may nest but not overlap`, + ); + } + } + } +} + +function checkNotes(notes: unknown, report: FlowSemanticReport) { + if (!Array.isArray(notes)) return; + const ids = new Set(); + notes.forEach((n: any, i: number) => { + const at = `/value/notes/${i}`; + checkColor(at, n?.color, report); + if (typeof n?.id === "string") { + if (ids.has(n.id)) { + report.warnings.push(`${at} reuses the note id '${n.id}'`); + } + ids.add(n.id); + } + if (n?.type === "group") { + report.warnings.push( + `${at} is a 'group' note, which is deprecated: use value.groups instead`, + ); + } else if (!n?.position || !n?.size) { + report.warnings.push( + `${at} has no position or size, so the editor places it at the origin and cannot resize it`, + ); + } + }); +} + +export function checkFlowSemantics(flow: unknown): FlowSemanticReport { + const report: FlowSemanticReport = { errors: [], warnings: [] }; + const value = (flow as any)?.value; + if (!value || typeof value !== "object") return report; + + const lists: ModuleList[] = []; + const agents: { pointer: string; module: any }[] = []; + walkModules(value.modules, "/value/modules", lists, agents); + + for (const agent of agents) { + checkAgentTools(agent, report); + } + checkGroups(value.groups, lists, report); + checkNotes(value.notes, report); + return report; +} diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index d031c16b3f..7029c851e0 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -36,6 +36,7 @@ import { getScriptBasePathFromModulePath, } from "../../utils/resource_folders.ts"; import { isFilesetResource } from "../../utils/utils.ts"; +import { checkFlowSemantics } from "./flow_semantics.ts"; import { exts, findContentFile, @@ -823,6 +824,16 @@ export async function runLint( ...formatYamlDiagnostics(result.parsed), ...result.errors.map((error) => formatValidationError(error)), ]; + if (target.type === "flow") { + const semantics = checkFlowSemantics(result.parsed?.data); + fileErrors.push(...semantics.errors); + warnings.push( + ...semantics.warnings.map((message) => ({ + path: normalizedPath, + message, + })), + ); + } if (fileErrors.length > 0) { issues.push({ path: normalizedPath, @@ -979,7 +990,7 @@ async function lintWatch(opts: LintOptions, directory?: string) { const command = new Command() .description( - "Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file", + "Validate Windmill flow, schedule, and trigger YAML files in a directory (including AI agent tool names and flow groups/notes), and report script metadata that has no deployable content file", ) .arguments("[directory:string]") .option("--json", "Output results in JSON format") diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 6185decc5a..9dfc26ca85 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -772,12 +772,20 @@ export function generateAgentsDocumentation( tables?: string[]; datatable?: string; schema?: string; + roles?: Record; } | undefined, ): string { const tables = data?.tables ?? []; const defaultDatatable = data?.datatable; const defaultSchema = data?.schema; + // A bare `wmill.datatable()` always means `main`, and a datatable under a role + // is only reachable through that role, so the example names both. + const role = defaultDatatable ? data?.roles?.[defaultDatatable] : undefined; + const datatableCall = `wmill.datatable('${defaultDatatable || "main"}'${ + role ? `, { role: '${role}' }` : "" + })`; + const tableRef = defaultSchema ? `${defaultSchema}.table` : "table"; return `# AI Agent Instructions @@ -791,7 +799,13 @@ This file contains **app-specific configuration** for this raw app instance. ${ defaultDatatable - ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ""}` + ? `**Default Datatable:** \`${defaultDatatable}\`${defaultSchema ? ` | **Default Schema:** \`${defaultSchema}\`` : ""}${ + role ? ` | **Role:** \`${role}\` (pass it to every \`wmill.datatable\` call)` : "" + }${ + defaultSchema + ? `\n\nWrite this app's tables as \`${defaultSchema}.\`, in migrations and queries: an unqualified name means the \`public\` schema.` + : "" + }` : "**No default datatable configured.** Set \`data.datatable\` in \`raw_app.yaml\` to enable database access." } @@ -833,8 +847,8 @@ const result = await backend.({ arg: 'value' }); **Query datatable (TypeScript):** \`\`\`typescript -const sql = wmill.datatable(); -const rows = await sql\`SELECT * FROM table WHERE id = \${id}\`.fetch(); +const sql = ${datatableCall}; +const rows = await sql\`SELECT * FROM ${tableRef} WHERE id = \${id}\`.fetch(); \`\`\` **SQL migrations:** Add \`.sql\` files to \`sql_to_apply/\`, run \`wmill app dev\`, then whitelist tables diff --git a/cli/src/guidance/core.ts b/cli/src/guidance/core.ts index 202a6feaad..6060f6b8a5 100644 --- a/cli/src/guidance/core.ts +++ b/cli/src/guidance/core.ts @@ -70,7 +70,8 @@ You are a helpful assistant that can help with Windmill scripts, flows, apps, an ## Important Notes - Every new entity MUST be created using the skills listed below. - Every modification of an entity MUST be done using the skills listed below. -- User MUST be asked where to create the entity. It can be in its user folder, under u/{user_name} folder, or in a new folder, /f/{folder_name}/. folder_name and user_name must be provided by the user. +- User MUST be asked where to create the entity: in their user space, \`u/{user_name}/\`, or in a folder, \`f/{folder_name}/\` (no leading \`/\`; a bare \`f/\` with no folder segment is invalid). folder_name and user_name must be provided by the user. +- A new folder needs its \`f/{folder_name}/folder.meta.yaml\`: create it with \`wmill folder new {folder_name}\`. \`wmill sync push\` refuses a folder without one for non-admins. ## Script Writing Guide @@ -82,6 +83,10 @@ For Workflow-as-Code scripts, use the \`write-workflow-as-code\` skill. You MUST use the \`write-flow\` skill to create or modify flows. When a new flow needs to be created, YOU run \`wmill flow new \` yourself (with \`--summary\` and optional \`--description\`) to scaffold the folder and \`flow.yaml\`, then edit \`flow.yaml\` to fill in modules and schema. Do NOT scaffold the folder + yaml by hand and do NOT tell the user to run \`wmill flow new\`. If path or summary are missing from the user's request, ask via \`AskUserQuestion\` (one call, all missing fields) — never invent them. See the \`write-flow\` skill for the procedure. +## Data Pipelines + +A data pipeline is NOT a flow: it is a set of independent scripts in one folder, marked \`pipeline\` and wired together by \`on\` / \`materialize\` annotations. When the user asks for a data pipeline (or to ingest, transform or materialize data across steps), you MUST use the \`write-pipeline\` skill and build pipeline scripts, not a flow. + ## Raw App Development You MUST use the \`raw-app\` skill to create or modify raw apps. diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 81895997b9..02f5d3a27d 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -33,6 +33,7 @@ export const SKILLS: SkillMetadata[] = [ { name: "schedules", description: "MUST use when configuring schedules." }, { name: "resources", description: "MUST use when managing resources." }, { name: "write-workflow-as-code", description: "MUST use when writing or modifying Windmill Workflow-as-Code scripts using workflow, task, step, sleep, approvals, taskScript, taskFlow, task_script, or task_flow." }, + { name: "write-pipeline", description: "MUST use when creating or modifying a data pipeline, a set of scripts marked `pipeline` and wired together by `on` / `materialize` annotations." }, { name: "cli-commands", description: "MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`." }, { name: "preview", description: "MUST use when opening the Windmill dev page / visual preview of a flow, script, or app. Triggers on words like preview, open, navigate to, visualize, see the flow/app/script, and after writing a flow/script/app for visual verification." }, ]; @@ -89,6 +90,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Ansible Windmill runs Ansible playbooks with \`ansible-playbook\`. A script is a single YAML @@ -229,6 +253,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Bash ## Structure @@ -329,6 +376,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # BigQuery Arguments use \`@name\` syntax. @@ -422,6 +492,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # TypeScript (Bun) Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. @@ -476,7 +569,7 @@ import * as wmill from "windmill-client"; **Prefer \`windmill-client\` over raw \`fetch\` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve \`fetch\` for calling *external* HTTP APIs that aren't Windmill. -The full \`windmill-client\` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method to use instead of guessing or falling back to \`fetch\`. +The full \`windmill-client\` API reference (every exported function and its signature) is included below — consult it for the exact method to use instead of guessing or falling back to \`fetch\`. ## Preprocessor Scripts @@ -494,7 +587,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -542,7 +637,6 @@ const result: wmill.S3Object = await wmill.writeS3File( ); \`\`\` - # TypeScript SDK (windmill-client) Import: import * as wmill from 'windmill-client' @@ -1218,6 +1312,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # TypeScript (Bun Native) Native TypeScript execution. Native scripts are Bun scripts that run on the native worker — a lightweight V8 isolate that exposes \`fetch\` and the JavaScript standard library — and can be heavily parallelized. Every script MUST start with \`//native\` on its first line so Windmill routes it to the native worker; without it the exact same script runs on the regular Bun worker. You may import npm packages and other Windmill scripts (e.g. \`./helper.ts\`) — imports are resolved and bundled just like a regular Bun script — as long as everything (your code and its dependencies) relies only on \`fetch\` and the standard library. Libraries that need Node/Bun runtime APIs (filesystem, \`node:*\` modules, child processes, native addons) will not work on the native worker; use the regular \`bun\` language for those. @@ -1269,7 +1386,7 @@ export async function main(url: string) { \`windmill-client\` works on the native worker (its calls go over \`fetch\`), so use it as the **preferred way to talk to Windmill** — reading resources/variables/states, running scripts and flows, and the S3 helpers below (\`loadS3File\`, \`loadS3FileStream\`, \`writeS3File\`, \`S3Object\`). It handles auth, the workspace, and the base URL for you. Reserve raw \`fetch\` for calling *external* HTTP APIs that aren't Windmill. -The full \`windmill-client\` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of hand-rolling a \`fetch\` against the Windmill API. +The full \`windmill-client\` API reference (every exported function and its signature) is included below — consult it for the exact method instead of hand-rolling a \`fetch\` against the Windmill API. ## Preprocessor Scripts @@ -1288,7 +1405,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -1338,7 +1457,6 @@ const result: wmill.S3Object = await wmill.writeS3File( ); \`\`\` - # TypeScript SDK (windmill-client) Import: import * as wmill from 'windmill-client' @@ -2014,6 +2132,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # C# The script must contain a public static \`Main\` method inside a class: @@ -2106,6 +2247,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # TypeScript (Deno) Deno runtime with npm support via \`npm:\` prefix and native Deno libraries. @@ -2162,7 +2326,7 @@ import * as wmill from "windmill-client"; **Prefer \`windmill-client\` over raw \`fetch\` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve \`fetch\` for calling *external* HTTP APIs that aren't Windmill. -The full \`windmill-client\` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of guessing or falling back to \`fetch\`. +The full \`windmill-client\` API reference (every exported function and its signature) is included below — consult it for the exact method instead of guessing or falling back to \`fetch\`. ## Preprocessor Scripts @@ -2180,7 +2344,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -2228,7 +2394,6 @@ const result: wmill.S3Object = await wmill.writeS3File( ); \`\`\` - # TypeScript SDK (windmill-client) Import: import * as wmill from 'windmill-client' @@ -2904,6 +3069,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # DuckDB Arguments are defined with comments and used with \`$name\` syntax: @@ -3030,6 +3218,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Go ## Structure @@ -3139,6 +3350,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # GraphQL ## Structure @@ -3235,6 +3469,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Java The script must contain a Main public class with a \`public static main()\` method: @@ -3324,6 +3581,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Microsoft SQL Server (MSSQL) Arguments use \`@P1\`, \`@P2\`, etc. @@ -3416,6 +3696,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # MySQL Arguments use \`?\` placeholders. @@ -3509,6 +3812,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # PHP ## Structure @@ -3617,6 +3943,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # PostgreSQL Arguments are obtained directly in the statement with \`$1::{type}\`, \`$2::{type}\`, etc. @@ -3708,6 +4057,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # PowerShell ## Structure @@ -3814,6 +4186,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Python ## Structure @@ -3897,7 +4292,7 @@ For preprocessor scripts, the function should be named \`preprocessor\` and rece from typing import TypedDict, Literal, Any class Event(TypedDict): - kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "amqp", "gcp", "azure"] body: Any headers: dict[str, str] query: dict[str, str] @@ -3948,7 +4343,6 @@ result: S3Object = wmill.write_s3_file( ) \`\`\` - # Python SDK (wmill) Import: import wmill @@ -4745,7 +5139,6 @@ async def parallel(items, fn, *, concurrency: Optional[int] = None) # partition: Partition number (from event['partition']) # offset: Message offset to commit (from event['offset']) def commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None - `, "write-script-rlang": `--- name: write-script-rlang @@ -4797,6 +5190,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # R ## Structure @@ -4933,6 +5349,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Rust ## Structure @@ -5059,6 +5498,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via \`results.step_id\` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. + # Snowflake Arguments use \`?\` placeholders. @@ -5189,7 +5651,6 @@ An input typed as a resource (\`format: resource-\` in the schema) takes t To open the flow visually in the dev page (graph + live reload), use the \`preview\` skill. Always **offer** it as a one-sentence next step (e.g. "Want me to open the visual preview?") rather than opening it automatically — opening the dev page has side effects (browser window, possibly a \`launch.json\` entry under MCP-preview branches) the user should consent to. If the user already asked to see/preview/visualize the flow in their original request, skip the offer and just invoke the skill. - # Windmill Flow Building Guide ## OpenFlow Schema @@ -5362,8 +5823,8 @@ names, so neither is name-checked at all — leave those summaries as they are. - Always set \`summary\`. It must be unique among that agent's tools, and must not be one of the reserved ids (\`do\`, \`bg\`, \`ctx\`, \`state\`, \`if\`, \`else\`, \`for\`, \`delete\`, \`while\`, \`new\`, \`in\`, \`failure\`, \`preprocessor\`, \`as\`, \`Input\`, \`Result\`, \`Trigger\`) -- A tool name outside that character set is rejected: flow write tools refuse it, and a flow that - reaches the worker with one fails every run with \`Invalid tool name\` +- A tool name outside that character set fails any run that offers the tool to the agent, with \`Invalid tool name\`. + \`wmill lint \` reports it before anything runs. - Tool \`id\` follows the same rules as any module ID — unique across the flow, underscores not spaces - \`description\` is optional free text telling the agent when and how to call the tool. Set it whenever the name alone does not make that obvious; it overrides the description derived from the @@ -5386,9 +5847,11 @@ names, so neither is name-checked at all — leave those summaries as they are. ## Loop Structure Rules +- A \`forloopflow\` runs its \`modules\` once per element of \`iterator\`, a javascript expression returning an array (e.g. \`results.get_items\`); \`parallel: true\` runs the iterations concurrently, and \`skip_failures: true\` carries on past a failed iteration - For \`whileloopflow\`, break the loop with a module-level \`stop_after_if\`: on the loop module itself, or on an inner step (required when that step carries state via its own \`results\` — see below) - \`stop_after_if\` is always a sibling of \`id\` and \`value\` on a flow module — never a direct key of the loop's \`value\` object - \`stop_after_all_iters_if\` is for checks after the whole loop finishes, not the normal per-iteration break condition +- \`stop_after_if\` is evaluated after each iteration: on the loop module, \`result\` is that iteration's result (what its last step returned); on an inner step, it is that step's result - \`flow_input.iter.value\` in a \`whileloopflow\` is just the iteration index (same number as \`flow_input.iter.index\`) — it never carries state, so \`flow_input.iter.value.\` is always undefined and a loop whose stop condition depends on it never terminates - To carry state across iterations, a step reads its own previous-iteration result via \`results.\` with a first-iteration fallback (e.g. \`results.b ?? flow_input.start\`) — but then the loop's \`stop_after_if\` MUST sit on that inner step, not on the loop module: a body that is exactly one plain step with the stop condition on the loop module runs on a fast path where \`results.\` is null on every iteration and the loop never terminates (bodies with 2+ steps, or whose single step has its own \`stop_after_if\`, retry or similar, resolve \`results\` across iterations regardless of stop placement) - For state that is just a counter, derive it from the index instead (e.g. \`flow_input.iter.index + 1\`) — that works in every configuration, including with \`stop_after_if\` on the loop module @@ -5526,6 +5989,7 @@ Incorrect shape (identity has no resume URLs — not a real approval): ## Branch Result Scope Rules +- A \`branchone\` runs the first of its \`branches\` whose \`expr\` is true, in order, and its \`default\` modules when none is; a \`branchall\` runs every branch (concurrently with \`parallel: true\`) - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch - Outside a \`branchone\`, do NOT reference ids of steps that only exist inside its branches or default branch. Use \`results.\` instead - Outside a \`branchall\`, do NOT reference ids of steps inside its branches. Use \`results.\` instead @@ -5585,6 +6049,41 @@ JavaScript transform (dynamic expression): - For flow inputs: Use type \`"object"\` with format \`"resource-{type}"\` (e.g., \`"resource-postgresql"\`) - For step inputs: Use static value \`"$res:path/to/resource"\` +## Reusing Existing Scripts and Flows + +Unless the user asked for new code, look for a workspace script or flow that already does a step's job before writing it, and reuse it by path instead of copying its logic into a rawscript: + +- a workspace script: \`type: script\` with \`path\` (e.g. \`f/folder/send_email\`) +- a workspace flow, run as a subflow: \`type: flow\` with \`path\` +- a Hub script: \`type: script\` with a \`hub///\` path + +The step's \`input_transforms\` must cover the reused item's inputs, so read its input schema first. +Find candidates in the local tree (a \`.script.yaml\` sits next to each script and holds its input schema, a \`flow.yaml\` in each flow folder) and on the workspace with \`wmill script list\` / \`wmill flow list\`; \`wmill script get \` and \`wmill flow get \` show an item's details. + +## Organizing Flows: Groups and Notes + +Groups and notes shape how a flow reads in the editor; neither changes what it does. + +**Segment every non-trivial flow into groups without waiting to be asked.** Whenever a flow has more than a couple of steps, or consecutive steps form a stage ("fetch", "transform", "notify"), put them in a group, and aim for every meaningful step to belong to one. Use notes sparingly, for flow-wide information that belongs to no span of steps: the flow's purpose, key assumptions, warnings, TODOs. One note is usually enough; never label a run of steps with a note, which is what a group is for. + +\`value.groups\` lists the groups, each spanning the steps from \`start_id\` to \`end_id\`: + +- \`start_id\`, \`end_id\` (required): ids of the group's first and last step; the same id for both makes a one-step group +- \`summary\`: the group's title +- \`note\`: markdown shown under the title +- \`color\`: one of \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\`, never a hex code or CSS color; leave it out and the editor picks one +- \`autocollapse\`: \`true\` shows the group collapsed by default + +The editor refuses to draw a flow whose groups break any of these rules: + +- \`start_id\` and \`end_id\` are steps of the same list: both top-level, or both in the same loop body or branch. A group can hold a loop or branch step whole, but cannot start outside one and end inside it +- \`start_id\` does not come after \`end_id\` in that list +- groups nest (one entirely inside another) but never partly overlap, and no two groups share both \`start_id\` and \`end_id\` +- groups hold ordinary steps only: never \`preprocessor\`, \`failure\`, \`Input\`, \`Result\`, \`Trigger\`, or an AI agent's tools + +\`value.notes\` lists sticky notes, each with a unique \`id\`, markdown \`text\`, a \`color\` from the same list, and \`type: free\`. The \`group\` note type is deprecated; use \`value.groups\` instead. +Give each note a \`position\` (\`{ x, y }\`) and a \`size\` (\`{ width, height }\`): the editor draws a note without them at the origin and cannot resize it. \`x: -400\` with \`width: 275\` places it beside the graph. + ## Final Structural Self-Check Before finalizing a flow, verify: @@ -5594,6 +6093,8 @@ Before finalizing a flow, verify: - any approval step has module-level \`suspend\` - no downstream step references inner branch step ids from outside the branch - every AI agent flowmodule tool has a unique \`summary\` made only of letters, numbers and underscores +- every group starts and ends on steps of the same list, start before end, nesting without partial overlap +- \`wmill lint \` reports no error ## S3 Object Operations @@ -5648,10 +6149,10 @@ Reference a specific resource using \`$res:\` prefix: } \`\`\` - ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\`: it is ignored when memory is off, and an older \`auto\` or\\n\`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\`: it is ignored when memory is off, and an older \`auto\` or\\n\`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -5822,7 +6323,7 @@ The \`data\` block in \`raw_app.yaml\` controls which tables the app can query. \`\`\`yaml data: datatable: main # Default datatable - schema: app_schema # Default schema (optional) + schema: app_schema # Schema the app's tables go in (optional); still write them as app_schema.
    tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema @@ -5867,6 +6368,8 @@ data: - main/users \`\`\` +A migration runs with no default schema, so a table outside \`public\` is created with its schema, \`CREATE TABLE IF NOT EXISTS app_schema.items (...)\`, listed as \`main/app_schema:items\`, and queried as \`app_schema.items\`. + ### Migration best practices - **Use idempotent SQL**: \`CREATE TABLE IF NOT EXISTS\`, etc. @@ -5876,8 +6379,9 @@ data: ## CLI Commands -Two commands you run yourself, not the user: +Commands you run yourself, not the user: - \`wmill app new\` — run it with flags, per the "Creating a Raw App" section above. +- \`wmill app lint \` — checks the app's structure and that it builds. Run it after editing, before offering a preview or a deploy; a bundle that compiles still says nothing about behavior, so a preview is what checks that. - \`wmill generate-metadata\` — (re)generates local lock files and refreshes \`wmill-lock.yaml\` content hashes; writes local files only (not a deploy). After adding or editing a runnable, offer it and run it on agreement — or automatically if the project's \`AGENTS.md\` opts into that (see "After creating a runnable" above). For the rest, tell the user which command fits their intent and let them run it — these deploy to the workspace, overwrite local files, or launch a long-running server, so the user should consent each time: @@ -5889,8 +6393,6 @@ For the rest, tell the user which command fits their intent and let them run it | \`wmill sync push\` | Deploy app to Windmill | | \`wmill sync pull\` | Pull latest from Windmill | - - # Windmill Raw Apps Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. @@ -5941,6 +6443,8 @@ Import the generated bindings and call the runnable like a function. \`./wmill\` | \`getJob(jobId)\` | a \`Job\` (\`{ type, success, result, duration_ms, ... }\`) | polling status without blocking | | \`streamJob(jobId, onUpdate?)\` | the final result, calling \`onUpdate\` per chunk | showing output as it is produced | +A runnable is always called with **one object** whose keys are its \`main\` parameters — \`main(user_id: string, limit: number)\` is called as \`backend.get_users({ user_id, limit })\`, never with positional arguments. A runnable without parameters is called with no argument. Resource and variable ids handed to the \`wmill\` client are paths (\`u//\` or \`f//\`). + Run and wait — the common case: \`\`\`tsx @@ -6014,9 +6518,10 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo ### Inline runnables -Inline runnables carry their own source code. For file-based raw apps, the runnable language is determined by the backend file extension. The script must expose a \`main\` function as its entrypoint. +Inline runnables carry their own source code, and must expose a \`main\` function as their entrypoint. +On disk, a runnable's language is determined by its backend file extension. -**TypeScript example** (\`backend/get_user.ts\`): +**TypeScript example** (runnable \`get_user\`): \`\`\`typescript import * as wmill from 'windmill-client'; @@ -6028,7 +6533,7 @@ export async function main(user_id: string) { } \`\`\` -**Python example** (\`backend/get_user.py\`): +**Python example** (runnable \`get_user\`): \`\`\`python import wmill @@ -6045,12 +6550,14 @@ An inline runnable runs as an ordinary Windmill job. \`import * as wmill from 'w **Don't read \`WM_TOKEN\` or \`BASE_INTERNAL_URL\` and build an API URL to \`fetch\`.** The client's own \`setClient\` already reads exactly those, and it also sets the credentials mode a raw app needs (\`WM_RAW_APP\` suppresses credentials, because a sandboxed bundle calls the API from an opaque origin that can never pair with \`Access-Control-Allow-Origin: *\`). Rebuilding that by hand drops the parts you can't see. Use \`wmill.*\` for everything Windmill, and \`fetch\` only for third-party APIs. -Prefer the \`wmill\` functions that appear in the SDK reference; for an endpoint none of them covers, the generated service classes (\`JobService\`, \`ScriptService\`, ...) are importable from \`windmill-client\`. What is not available is a name you guessed at: \`getBaseUrl\` and \`getWorkspaceToken\` are inventions, not API. +Use only \`wmill\` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (\`JobService\`, \`ScriptService\`, ...) are importable from \`windmill-client\`. What is not available is a name you guessed at: \`getBaseUrl\` and \`getWorkspaceToken\` are inventions, not API. ### Path runnables (script / flow / hubscript) When \`type\` is \`script\`, \`flow\`, or \`hubscript\`, the runnable just stores a \`path\` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at \`hub///\`) for a third-party service, and reference it instead of copying its logic. + ### Draft code vs deployed code This decides whether an app works before anything is deployed: @@ -6070,15 +6577,29 @@ Prefer a **path runnable of type \`flow\`** over an inline runnable that calls \ \`staticInputs\` is an optional \`Record\` for arguments not overridable from the frontend. Useful with path runnables to pre-fill some args while leaving the rest to the frontend caller. +## Who can open a deployed app + +A draft app is reachable by nobody; deploying is what exposes it, and its backend runnables with it. Otherwise only users with access to the app can open it, unless the app is opened to: + +- **anonymous** users: anyone with the URL, without logging in; +- **guests**: anyone the instance's identity provider authenticates, whether or not they belong to the workspace. + +Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. +\`public: true\` in \`raw_app.yaml\` deploys the app for anonymous users, and \`guests: true\` for guests. Remove the line and the next push closes the app again. + ## Data Tables -Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. +Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. ### Critical rules -1. **Whitelisted tables only**: a runnable can only query tables listed in the app's \`data.tables\` config. Tables not in this list are not accessible. -2. **Add tables before using**: queries against unlisted tables fail at runtime. When you introduce a new table, register it in \`data.tables\` first. -3. **Use the configured datatable/schema**: the app's \`data\` config sets the default datatable and schema; reference them consistently across runnables. +1. **Check what exists first**: look up the workspace's data tables and their tables before designing storage, and reuse a suitable table rather than creating another. Never assume a \`main\` data table exists. +2. **Whitelisted tables only**: a runnable can only query tables listed in the app's \`data.tables\` config. Queries against unlisted tables fail at runtime, so register a new table there before using it. +3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. +4. **Qualify table names**: an unqualified name means the \`public\` schema, so write every other table as \`schema.table\`, in table creation and queries alike. The app's \`data\` config sets the default datatable and schema its tables go in; use them consistently across runnables. +5. **Pass the role**: when the app's \`data.roles\` gives a data table a role, every \`wmill.datatable\` call on it passes that role (\`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python). The role only reaches what it was granted, so a query outside it fails with \`permission denied\`. + +\`wmill datatable list\` lists the workspace's data tables. Create or change tables with a migration in \`sql_to_apply/\` (see "SQL Migrations" above), then add them to \`data.tables\` in \`raw_app.yaml\`. ### Querying in TypeScript (Bun/Deno) @@ -6313,6 +6834,14 @@ Reference variables in resource values: - \`$var:u/username/name\` - User variable - \`$var:f/folder/name\` - Folder variable +## Secrets + +Never put a secret (password, API key, token) inline in a resource value. Store it in a secret variable and reference that variable as \`$var:\`. + +- \`$var:\` is a reference, not a value: it resolves to the variable's value at run time. Never invent a value for a variable. +- A resource that references a variable needs the variable to exist first, so create or deploy the variable before the resource. +- A secret's plaintext never goes in a file of the repo: a \`.variable.yaml\` holding it would be committed. Ask the user to create the secret on the workspace instead, e.g. \`wmill variable add '' \` (a secret by default), then reference it by path. + ## Resource References Reference other resources: @@ -6594,14 +7123,13 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than \`script p Use \`wmill resource-type list --schema\` to discover available resource types. -Workflow-as-Code files use the normal script CLI workflow. There are no separate WAC deploy commands. - # Windmill Workflow-as-Code Writing Guide ## Scope Use this guide when writing or modifying Windmill Workflow-as-Code (WAC) scripts. WAC is authored as a Windmill script and deployed with the normal script workflow. It is not an OpenFlow YAML flow. +Workflow-as-Code files use the normal script CLI workflow. There are no separate WAC deploy commands. Supported WAC authoring targets: - Bun TypeScript scripts that import from \`windmill-client\` @@ -6786,7 +7314,6 @@ TypeScript: avoid broad \`try/catch\` around WAC SDK calls. The SDK uses an inte A caught failure reads the same whether it came from a task or from a \`step()\`, and the same in the round that ran the failing body as in every round replaying it. It carries \`step_key\`, \`child_job_id\` (absent for a \`step()\`, which runs in the workflow job and has no child job), a \`message\` that is the failure's own message, and \`result\` = \`{"error": {"name", "message", "stack"?, "extra"?}}\`. \`name\`, \`message\` and \`stack\` are the fields that read the same whichever side failed; \`name\` and \`message\` are always there, \`stack\` only when the failure had a traceback to give. \`extra\` carries the failure's own custom fields (an exception's attributes, an error's properties) and is best-effort: it is absent when there were none, and a task can report entries a step does not, so read it defensively and don't branch on its absence. \`extra\` is dropped when it is too large to keep in the checkpoint, and \`extra_omitted: true\` says so — absent \`extra\` with no \`extra_omitted\` means the failure simply had no custom fields. Branch on those, not on the original exception type: the workflow body re-runs from the top every round and a replay rebuilds the failure from the checkpoint, so nothing outside that record survives. Python raises \`TaskError\`; TypeScript throws an \`Error\` named \`TaskError\` carrying the same fields. Nothing is chained onto \`__cause__\` / \`cause\` — the traceback is in \`result.error.stack\`, and is also printed to the job log when the step fails. - ## TypeScript Workflow-as-Code API (windmill-client) Import: \`import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"\` @@ -7155,6 +7682,132 @@ def get_approval_urls(step_key: str = 'approval', approver: str = None) -> dict # results = await parallel(items, process, concurrency=5) async def parallel(items, fn, *, concurrency: Optional[int] = None) \`\`\` +`, + "write-pipeline": `--- +name: write-pipeline +description: MUST use when creating or modifying a data pipeline, a set of scripts marked \`pipeline\` and wired together by \`on\` / \`materialize\` annotations. +--- + +# Data pipeline authoring + +A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at \`/pipeline/\`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. + +## Default to DuckDB + DuckLake + +A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake:///
    \` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: + +- \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). +- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. + +Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. + +## Storage prerequisites + +A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. +\`wmill ducklake list\` lists them. +Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if there is none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: + +- a workspace **admin** sets it up in Workspace settings → Object Storage (add an S3/Azure/GCS storage), then adds a DuckLake catalog on top of it; +- anyone **without admin rights** should ask a workspace admin to configure object storage + a DuckLake catalog. + +Never hand back a DuckLake pipeline that cannot run without flagging the missing storage and pointing to who sets it up. + +## What makes a script a pipeline node + +A script joins the pipeline when its source begins with the \`pipeline\` annotation as a top-of-file comment, **written in the script's own comment syntax** — \`//\` for TS/JS (bun), \`--\` for SQL (DuckDB/Postgres), \`#\` for Python/Bash. So it's \`-- pipeline\` in a DuckDB node, \`# pipeline\` in a Python node, \`// pipeline\` in a bun node. Every annotation below uses that same prefix (the \`//\` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: + +- \`// on \` — declares an execution-DAG **input** (what triggers/feeds this node). \`\` is either: + - an **asset URI** (the node runs when that asset is produced upstream): \`ducklake://main/orders\`, \`datatable://main/users\`, \`$res:f/folder/my_resource\`, \`volume://name/path\`, or an S3 object (see the S3 storage-form rule below). + - a **native trigger kind**: \`schedule\`, \`webhook\`, \`email\`, \`kafka\`, \`mqtt\`, \`amqp\`, \`nats\`, \`postgres\`, \`sqs\`, \`gcp\`, or \`data_upload\` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. **\`data_upload\` is special**: there is no trigger row — the node instead declares an **\`S3Object\` input parameter** fed by the auto-generated upload picker; it never hard-codes a key. Any language can be the \`data_upload\` node: + - Python (has \`import wmill\`): \`def main(file: wmill.S3Object):\` then \`wmill.load_s3_file(file)\`; TS (has \`import * as wmill from "windmill-client"\`): \`export async function main(file: wmill.S3Object)\`. Qualify the type as \`wmill.S3Object\` (or add \`from wmill import S3Object\` / \`import { S3Object } from "windmill-client"\`) — a bare \`S3Object\` is undefined. + - **DuckDB** takes the s3object arg via a \`-- $ (s3object)\` declaration and reads it directly, so a single DuckDB node can ingest **and** materialize: + \`\`\` + -- pipeline + -- on data_upload + -- materialize ducklake://main/raw_uploads + -- $file (s3object) + SELECT * FROM read_csv($file) + \`\`\` +- **Outputs** are inferred from what the body writes — a \`CREATE TABLE\`, a \`wmill.writeS3File(...)\`, a DuckLake/datatable write. To declare a managed output explicitly, use \`// materialize \`. +- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` one), \`// measure = [where ]\` and \`// dimension = \` (see "Declared metrics" below). + +## S3 object wiring (storage form matters) + +An \`s3://\` URI's first slashes select the **storage**, not part of the key — get this wrong and the producer/consumer edge silently won't connect: + +- \`s3:///\` (**triple** slash, empty first segment) = the **default** workspace storage. A downstream node reading or triggering on that object uses \`s3:///\` — e.g. DuckDB \`-- on s3:///orders/2024.parquet\` and \`read_parquet('s3:///orders/2024.parquet')\`. +- \`s3:///\` (**double** slash, non-empty first segment) = a **named secondary** storage called \`\` — so \`s3://ingest/x\` means storage \`ingest\`, key \`x\`, NOT key \`ingest/x\`. Only use this when the object genuinely lives in a configured secondary storage; never invent a bucket/storage name for a default-storage object (it breaks the edge). + +To make the producer side visible to lineage, a Python/TS node MUST pass the **\`S3Object\` form**, not a bare key string: Python \`wmill.write_s3_file(wmill.S3Object(s3=""), data)\` (or the import-free dict \`{"s3": ""}\`), TS \`wmill.writeS3File({ s3: "" }, data)\`. That records the default-storage asset \`/\`, which a downstream \`s3:///\` reader connects to (same key both sides). A bare \`write_s3_file("", ...)\` records **no** asset and produces **no** edge. Add \`storage=""\` only for a named secondary storage. + +The key must be a **string literal** — the graph parser is static and cannot follow a variable, f-string, or computed path, so \`write_s3_file(wmill.S3Object(s3=key_var), ...)\` records no edge. Inline the literal (\`s3="events/user_events.parquet"\`) on both the writing and reading node. The same rule applies to every asset URI in an annotation or SDK call (\`ducklake://\`, \`datatable://\`, \`s3://\`): write them literally, not via a variable. + +## Materialize (the managed output) + +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
    \`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> +> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. + +A managed \`// materialize ducklake:///
    \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. (The opposite holds for the \`dbt://\` target above: there the node writes its own DDL and the strategies below do not apply.) Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- \`// materialize append\` → INSERT-append rows (incremental); +- \`// materialize key=\` → merge/upsert on \`\`. + +\`// materialize manual \` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table). The \`{partition}\` token, usable in any asset URI **and** in the body SQL, is replaced at run time by the current partition's **identity string**: + +- To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp in exactly the identity format the runtime uses for daily/hourly/weekly/monthly, so it always matches; never hand-write a \`strftime\` format. +- Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at run time. +- For \`dynamic\` partitioning the identity is the caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. + +\`materialize\` is an output **declaration** on a node — not a command. There is no "materialize run". + +## Declared metrics (\`measure\` / \`dimension\`) + +On a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents reuse the definition instead of re-deriving it and silently disagreeing. + +- Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates share one GROUP BY. +- DuckLake-only, and only meaningful next to \`// materialize\`. +- Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures — an obvious \`count(*)\` earns nothing. +- To use a metric another node declares, read that node and reuse its exact expression rather than guessing it. + +## How to build one + +1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. +2. Write each node as its own script. Default to \`duckdb\` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick \`postgresql\`, \`bun\`, or \`python3\` only when that section says the work calls for it. +3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Don't deploy nodes unless the user asks to. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +Locally: + +- create each node with \`wmill script new f// \`, then write its body; +- \`wmill pipeline show --local\` draws the graph from your working tree — check that every edge you meant to form is there; +- \`wmill pipeline dev \` live-previews the pipeline, and \`wmill pipeline run --local\` runs the cascade from local files without deploying (\`--dry-run\` prints the plan first); +- a trigger such as \`// on schedule\` only declares the binding: the schedule or trigger itself is created separately (see the \`schedules\` and \`triggers\` skills). + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node \`f/sales/orders_ingest\` (runs on a schedule, materializes a DuckLake table): + +\`\`\`sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3:///raw/orders/*.csv') +\`\`\` + +Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup): + +\`\`\`sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +\`\`\` `, "cli-commands": `--- name: cli-commands @@ -7538,7 +8191,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file +Validate Windmill flow, schedule, and trigger YAML files in a directory (including AI agent tool names and flow groups/notes), and report script metadata that has no deployable content file **Arguments:** \`[directory:string]\` diff --git a/cli/test/lint_command_unit.test.ts b/cli/test/lint_command_unit.test.ts index 10ecd80bdb..4afe905b3f 100644 --- a/cli/test/lint_command_unit.test.ts +++ b/cli/test/lint_command_unit.test.ts @@ -386,3 +386,167 @@ raw_string: false ).toBeTruthy(); }); }); + +async function lintFlow(tempDir: string, value: string) { + await mkdir(`${tempDir}/f/sem/check.flow`, { recursive: true }); + await writeFile( + `${tempDir}/f/sem/check.flow/flow.yaml`, + `summary: Semantic checks\nvalue:\n${value}`, + "utf-8", + ); + return await runLint({} as any, tempDir); +} + +test("lint: rejects AI agent tool names the worker refuses", async () => { + await withTempDir(async (tempDir) => { + const report = await lintFlow( + tempDir, + ` modules: + - id: agent + value: + type: aiagent + input_transforms: {} + tools: + - id: t_spaced + summary: Search docs + value: { tool_type: flowmodule, type: script, path: f/sem/search, input_transforms: {} } + - id: t_missing + value: { tool_type: flowmodule, type: script, path: f/sem/search, input_transforms: {} } + - id: t_first + summary: get_user + value: { tool_type: flowmodule, type: script, path: f/sem/get_user, input_transforms: {} } + - id: t_dup + summary: get_user + value: { tool_type: flowmodule, type: script, path: f/sem/get_user, input_transforms: {} } + - id: t_reserved + summary: if + value: { tool_type: flowmodule, type: script, path: f/sem/branch, input_transforms: {} } + - id: t_mcp + summary: "" + value: { tool_type: mcp, resource_path: f/sem/mcp } + - id: t_web + summary: Web Search + value: { tool_type: websearch } + - id: linked + value: + type: aiagent + agent: f/sem/support_agent + input_transforms: {} + tools: + - id: t_ignored + summary: Stale name + value: { tool_type: flowmodule, type: script, path: f/sem/search, input_transforms: {} } +`, + ); + + const errors = report.issues.flatMap((i) => i.errors); + expect(errors).toEqual([ + "/value/modules/0/value/tools/0 tool name 'Search docs' may only contain letters, numbers and underscores (a run that offers this tool fails with 'Invalid tool name')", + "/value/modules/0/value/tools/1 has no summary: a tool's summary is the name the agent calls it by", + "/value/modules/0/value/tools/3 tool name 'get_user' is already used by tool 2 of this agent", + ]); + expect(report.warnings.map((w) => w.message)).toEqual([ + "/value/modules/0/value/tools/4 tool name 'if' is a reserved id; the flow editor refuses it", + ]); + expect(report.exitCode).toEqual(1); + }); +}); + +test("lint: rejects flow groups the editor cannot draw", async () => { + await withTempDir(async (tempDir) => { + const step = (id: string) => + ` - id: ${id}\n value: { type: identity }\n`; + const report = await lintFlow( + tempDir, + ` modules: +${step("a")}${step("b")}${step("c")} - id: loop + value: + type: forloopflow + iterator: { type: javascript, expr: "[1]" } + skip_failures: false + modules: + - id: inner + value: { type: identity } + groups: + - { start_id: c, end_id: a } + - { start_id: a, end_id: inner } + - { start_id: a, end_id: gone } + - { start_id: a, end_id: b } + - { start_id: b, end_id: c } + - { start_id: a, end_id: b } +`, + ); + + expect(report.issues.flatMap((i) => i.errors)).toEqual([ + "/value/groups/0 starts at 'c', which comes after its end 'a'", + "/value/groups/1 starts at 'a' and ends at 'inner', which are not in the same branch", + "/value/groups/2 references 'gone', which is not a step of this flow", + "/value/groups/5 spans the same steps as another group (a to b)", + "/value/groups/4 partly overlaps /value/groups/3: groups may nest but not overlap", + ]); + }); +}); + +test("lint: accepts nested groups and flags notes the editor renders wrong", async () => { + await withTempDir(async (tempDir) => { + const report = await lintFlow( + tempDir, + ` modules: + - { id: a, value: { type: identity } } + - { id: b, value: { type: identity } } + - id: loop + value: + type: forloopflow + iterator: { type: javascript, expr: "[1]" } + skip_failures: false + modules: + - { id: x, value: { type: identity } } + - { id: y, value: { type: identity } } + groups: + - { summary: Fetch, start_id: a, end_id: loop, color: blue } + - { summary: Prep, start_id: a, end_id: b } + - { summary: Body, start_id: x, end_id: y, color: "#ff0000" } + notes: + - { id: n1, text: Purpose, color: yellow, type: free, position: { x: -400, y: 0 }, size: { width: 275, height: 80 } } + - { id: n2, text: Unplaced, color: yellow, type: free } + - { id: n3, text: Old, color: gray, type: group, contained_node_ids: [a] } +`, + ); + + expect(report.issues).toEqual([]); + expect(report.warnings.map((w) => w.message)).toEqual([ + "/value/groups/2 color '#ff0000' renders unstyled; use one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray", + "/value/notes/1 has no position or size, so the editor places it at the origin and cannot resize it", + "/value/notes/2 is a 'group' note, which is deprecated: use value.groups instead", + ]); + expect(report.exitCode).toEqual(0); + }); +}); + +test("lint: reports a flow with malformed collections instead of aborting the run", async () => { + await withTempDir(async (tempDir) => { + await mkdir(`${tempDir}/f/sem/other.flow`, { recursive: true }); + await writeFile( + `${tempDir}/f/sem/other.flow/flow.yaml`, + `summary: Other\nvalue:\n modules: {}\n`, + "utf-8", + ); + const report = await lintFlow( + tempDir, + ` modules: + - id: route + value: { type: branchall, branches: {} } + - id: agent + value: { type: aiagent, input_transforms: {}, tools: {} } + groups: {} + notes: {} +`, + ); + + expect(report.issues.map((i) => i.path).sort()).toEqual([ + "f/sem/check.flow/flow.yaml", + "f/sem/other.flow/flow.yaml", + ]); + expect(report.exitCode).toEqual(1); + }); +}); diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 634341b966..7964f1a908 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -19,6 +19,7 @@ import { type AppDatatableElement } from '../context' import { appDatatableRole, sdkDatatableCall } from '$lib/components/raw_apps/dataTableRefUtils' +import { RAW_APP_BASE } from '$system_prompts' // Backend runnable types export type BackendRunnableType = 'script' | 'flow' | 'hubscript' | 'inline' @@ -937,36 +938,14 @@ export function prepareAppSystemMessage(customPrompt?: string): ChatCompletionSy )}. Always pass that role when calling \`wmill.datatable\` on them, as in the examples. The role only reaches what it was granted, so a query on a table it lacks privileges on fails with \`permission denied\`.` : '' - let content = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend. + // Domain guidance for raw apps (structure, runnables, data tables, access) is RAW_APP_BASE, + // shared with global mode and the CLI's raw-app skill; this prompt adds only the app editor's + // tools and this app's data-table policy. + let content = `You are a helpful assistant that creates and edits apps on the Windmill platform. Apps are defined as a collection of files that contains both the frontend and the backend; the reference below describes how they work. The sections after it cover this editor's tools and this app's own configuration, which take precedence over the reference's generic examples. Frontend files are managed separately from backend runnables, and inline backend runnables are TypeScript (Bun) or Python. -## App Structure +${RAW_APP_BASE} -### Frontend -- The frontend is bundled using esbuild, with entrypoint \`index.tsx\` for React and \`index.ts\` for Svelte and Vue -- The entrypoint is also the **mount** entrypoint: nothing is auto-rendered, so it must mount a top-level \`App\` into \`#root\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim: - \`\`\`tsx - import React from 'react' - import { createRoot } from 'react-dom/client' - import App from './App' - - createRoot(document.getElementById('root')!).render() - \`\`\` - (Svelte \`index.ts\`: \`mount(App, { target: document.getElementById('root')! })\`; Vue \`index.ts\`: \`createApp(App).mount('#root')\`.) -- **Never replace the entrypoint with a bare component.** A component that is defined but never mounted renders a blank screen with **no error** — it never executes, so nothing throws. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. -- Frontend files are managed separately from backend runnables -- The \`wmill.d.ts\` file is generated automatically from the backend runnables shape -- Begin every React file (\`.tsx\`/\`.jsx\`) that uses JSX with \`import React from 'react'\`. Raw apps bundle with the classic JSX transform, so \`React\` must be in scope wherever JSX is used — a missing import compiles fine but throws \`React is not defined\` at runtime. - -### Backend -Backend runnables can be of different types: -- **inline**: Custom code written directly in the app (TypeScript/Bun or Python) -- **script**: Reference to a workspace script by path -- **flow**: Reference to a workspace flow by path -- **hubscript**: Reference to a hub script by path - -Frontend calls backend using \`await backend.(args...)\`. - -For inline scripts, the code must have a \`main\` function as its entrypoint. +# App editor ## Available Tools @@ -1004,13 +983,11 @@ Use \`patch_file\` for small, localized edits when you can copy an exact snippet ## Data Storage with Data Tables -**When the app needs to store or persist data, you MUST use datatables.** Datatables provide a managed PostgreSQL database that integrates seamlessly with Windmill apps, with near-zero setup and workspace-scoped access. - -### Key Principles +Persist app data in data tables, following "Data Tables" in the reference above. 1. **Always check existing tables first**: Use \`list_datatables()\` to see what tables are already available. If a suitable table exists, **always reuse it** rather than creating a new one. For dashboards that only show available tables or row counts, \`list_datatables()\` is enough. Only call \`get_datatable_table_schema()\` for tables whose column names/types you need. -2. **CRITICAL: Create tables ONLY via exec_datatable_sql tool**: When you need to create a new table, you MUST use the \`exec_datatable_sql\` tool with the \`new_table\` parameter. **NEVER** create tables inside backend runnables using SQL queries - this will not register the table properly and it won't be available for future use. +2. **CRITICAL: Create tables ONLY via exec_datatable_sql tool**: When you need to create a new table, you MUST use the \`exec_datatable_sql\` tool with the \`new_table\` parameter, which also registers the table with the app. **NEVER** create tables inside backend runnables. \`\`\` exec_datatable_sql({ datatable_name: "main", @@ -1019,17 +996,9 @@ Use \`patch_file\` for small, localized edits when you can copy an exact snippet }) \`\`\` -3. **Use schemas to organize data**: Use PostgreSQL schemas to organize tables logically. Reference schemas with \`schema.table\` syntax. +### Accessing this app's data tables from backend runnables -4. **Use datatables for**: - - User data, settings, preferences - - Application state that needs to persist - - Lists, records, logs, history - - Any data the app needs to store and retrieve - -### Accessing Data Tables from Backend Runnables - -Backend runnables should only perform **data operations** (SELECT, INSERT, UPDATE, DELETE) on **existing tables**. Never use CREATE TABLE, DROP TABLE, or ALTER TABLE inside runnables. +For this app's tables, use these calls rather than the generic \`wmill.datatable()\` examples in the reference: they carry the app's data table, role and schema. **TypeScript (Bun) example**: \`\`\`typescript @@ -1106,31 +1075,6 @@ When creating a backend runnable with \`set_backend_runnable\`: } \`\`\` - -Windmill expects all backend runnable calls to use an object parameter structure. For example for: -\`\`\`typescript -export async function main(arg1: string, arg2: string, arg3: number, arg4: { field1: string, field2: number }) { - ... -} -\`\`\` - -You would call it like this: -\`\`\`typescript -await backend.myFunction({ arg1: 'value1', arg2: 'value2', arg3: 3, arg4: { field1: 'value1', field2: 2 } }) -\`\`\` -If the runnable has no parameters, you can call it without an object: -\`\`\`typescript -await backend.myFunction() -\`\`\` - -When you are using the windmill-client, do not forget that as id for variables or resources, those are path that are of the form \'u//\' or \'f//\'. - -Besides \`backend\`, the generated \`./wmill\` module exports \`backendAsync.(args)\` (resolves the job id as a string), \`waitJob(jobId)\` (resolves that job's result, rejects if it failed), \`getJob(jobId)\` (the current job state, for rendering progress) and \`streamJob(jobId, onUpdate)\`. Use \`backendAsync\` + \`waitJob\`/\`getJob\` for long-running work — never hand-write a runnable that polls job status, and never \`fetch\` the Windmill API from frontend code, which holds no token. - -A \`script\`/\`flow\` runnable runs the DEPLOYED item at that path, and so do \`wmill.runFlowAsync\`/\`wmill.runScriptByPath\` called inside a runnable — a draft is invisible to them, so an app pointed at an undeployed flow fails at runtime. The app itself does not need deploying — the preview runs its draft — so the fix is to deploy that one referenced item, not the whole change set. Say so instead of working around it, and never reimplement the flow inline to dodge the deployment. An \`inline\` runnable runs the app's own code and needs nothing deployed. - -Inside an inline runnable the \`wmill\` client configures itself from the job environment: don't read \`WM_TOKEN\` or \`BASE_INTERNAL_URL\` and build an API URL by hand — the client already does that, plus the credentials mode a raw app needs. Only call \`wmill\` functions that actually exist; \`getBaseUrl\` and \`getWorkspaceToken\` are inventions. - ## Instructions 1. Use the smallest context needed. If the target file or runnable is clear, inspect only that item with \`get_frontend_file(path)\` or \`get_backend_runnable(key)\`. diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index 96b2858957..4adbe1634b 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -854,7 +854,9 @@ export const flowTools: Tool[] = [ ] export function prepareFlowSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { - // Get base flow documentation from centralized prompts (includes FLOW_BASE, OPENFLOW_SCHEMA, RESOURCE_TYPES) + // Flow domain guidance (module shapes, loops, groups, AI agent tools…) is FLOW_BASE, shared with + // global mode and the CLI's write-flow skill: it belongs in system_prompts/base/flow-base.md. + // This prompt adds only the flow editor's tools. const flowBaseContext = getFlowPrompt() // Chat-specific tool instructions @@ -923,16 +925,8 @@ Use the \`set_flow_json\` tool to set the entire flow structure at once. Provide - \`schema\`: Flow input schema in JSON Schema format (optional) - \`preprocessor_module\`: Special module that runs before \`modules\` (optional, separate from \`modules\`) - \`failure_module\`: Special module that runs on failure (optional, separate from \`modules\`) -- \`groups\`: Array of semantic groups for organizing modules in the editor (optional, but **strongly recommended** — proactively segment any non-trivial flow into groups so it reads clearly; don't wait to be asked). Each group has \`summary\` (display name), \`note\` (markdown description shown below the group header — attached directly to the group, not a separate sticky note), \`autocollapse\`, \`start_id\`, \`end_id\`, and \`color\`. \`start_id\` and \`end_id\` must reference existing module IDs in the flow (not \`preprocessor\` or \`failure\`). \`color\` MUST be one of these exact names: \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\` — do NOT use hex codes, CSS colors, or any other strings. Omit \`color\` entirely if no preference and the editor will assign one automatically. Groups do not affect execution — they provide naming and collapsibility in the editor. Pass \`null\` to clear existing groups. -- \`notes\`: Array of free-floating sticky notes shown in the editor (optional). Each note has \`id\` (unique string), \`text\` (markdown content), \`color\` (same palette as groups: \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\` — never hex codes), and optional \`position\` {x, y} / \`size\` {width, height} (omit both — the editor auto-places and sizes the note). Always set \`type\` to \`free\`. The \`group\` note type is **deprecated** — do not create group notes; use the \`groups\` field to segment a flow instead. Notes are documentation only and do not affect execution. Pass \`null\` to clear existing notes. - -### When to use notes vs groups - -**Strongly prefer \`groups\` to organize flows.** Groups are the primary way to make a flow readable: whenever a flow has more than a couple of steps, or any time consecutive steps form a logical stage (e.g. "fetch", "transform", "notify"), segment them into \`groups\`. Each group spans a range of steps (\`start_id\`..\`end_id\`), carries its own \`summary\`, \`note\` (markdown under the group header), and \`color\`, and can be collapsed. Proactively add or update groups when building or restructuring a flow — do not wait to be asked. Aim for every meaningful step to belong to a semantic group. - -- **\`groups\` (default, use liberally):** segment a flow into labelled semantic sections. This is the main organizational tool — reach for it on essentially any non-trivial flow, not just "complex" ones. -- **\`notes\` (free sticky notes, use sparingly):** reserve for important flow-wide information that does not belong to a specific span of steps — overall purpose, key assumptions, warnings, or TODOs. Usually a single note is enough; do not use notes to label sequences of steps (that is what \`groups\` are for). -- Do **not** use \`group\`-type notes (deprecated) — \`groups\` is the supported way to group steps. +- \`groups\`: Array of semantic groups (optional, but add them to any non-trivial flow without being asked; see "Organizing Flows: Groups and Notes" below). Pass \`null\` to clear existing groups. +- \`notes\`: Array of free-floating sticky notes (optional; see the same section). Pass \`null\` to clear existing notes. **Example - Simple flow:** \`\`\`javascript diff --git a/frontend/src/lib/components/copilot/chat/flow/openFlow.json b/frontend/src/lib/components/copilot/chat/flow/openFlow.json index b8f44d2635..3bbd043a26 100644 --- a/frontend/src/lib/components/copilot/chat/flow/openFlow.json +++ b/frontend/src/lib/components/copilot/chat/flow/openFlow.json @@ -1 +1 @@ -{"openapi":"3.0.3","info":{"version":"1.813.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file +{"openapi":"3.0.3","info":{"version":"1.818.0","title":"OpenFlow Spec","contact":{"name":"Ruben Fiszel","email":"ruben@windmill.dev","url":"https://windmill.dev"},"license":{"name":"Apache 2.0","url":"https://www.apache.org/licenses/LICENSE-2.0.html"}},"paths":{},"externalDocs":{"description":"documentation portal","url":"https://windmill.dev"},"components":{"schemas":{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution — modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind"}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type"}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type"}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw",null]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw",null]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}]},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime — including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}}} \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 5664f2349d..84715538b2 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -2296,6 +2296,9 @@ async function listWorkspaceItems( return items } +// The get_instructions builders below add global mode's tools and draft rules on top of the shared +// reference each ends with. Windmill domain guidance belongs in that reference +// (system_prompts/base, shared with the other chat modes and the CLI's skills), not here. function getScriptInstructions(language: ScriptLang | undefined): string { const selected = language ?? 'bun' const note = language @@ -2333,18 +2336,9 @@ async function getFlowInstructions(workspace: string | undefined): Promise/main.{ts|py}\` from the file tools, but you create or update them via \`write_app_runnable\` / \`delete_app_runnable\` (which take the runnable shape directly: \`{ name, type, inlineScript?, path?, staticInputs? }\`). - \`/wmill.d.ts\` (or \`wmill.ts\`) is generated automatically from the backend runnables — never write it directly. - Inline runnables only support \`bun\` or \`python3\` in chat. Path runnables (\`script\`/\`flow\`/\`hubscript\`) reference an existing item. -- Inline runnables run the app's DRAFT code, so they work in the preview with nothing deployed. Path runnables — and \`wmill.runFlow*\` / \`runScriptByPath\` called from inside any runnable — run the DEPLOYED item at that path, and a draft is invisible to them. An app wired to a flow you just drafted does nothing until that flow is deployed — but the APP does not have to be deployed for that: the preview runs its draft. So offer to deploy just the referenced flow/script with deploy_workspace_item and leave the app a draft the user keeps testing in the preview; don't route a one-item dependency deploy through the compare page, and don't ask them to deploy the app unless they want to ship it. Never dodge it by reimplementing the flow inside an inline runnable — that leaves two copies of the same logic and an app that ignores the flow they asked for. +- When a path runnable points at a draft (see "Draft code vs deployed code" below), offer to deploy just that flow/script with deploy_workspace_item, not through the compare page. ${sdkLine} - Use \`deploy_workspace_item\` after explicit user deploy intent. The deploy tool bundles JS/CSS before saving the raw app. - Use \`read_workspace_item\` with \`type: 'app'\` for a metadata summary (file paths and runnable list, no contents). Use \`read_app_file\` to read an individual file; large files are truncated to a head slice, so pass \`offset\`/\`limit\` to page through the rest rather than re-reading the whole file. - To find where a symbol or string lives across the app, call \`search_app\` (greps every frontend file and inline runnable, returns matching \`file:line\` rows) instead of reading files one by one — then \`read_app_file\` only the ranges you need. The loop is list (\`read_workspace_item\`) → locate (\`search_app\`) → inspect (\`read_app_file\` with \`offset\`/\`limit\`). -- Note: the authoring reference below mentions the CLI on-disk layout (\`backend/.\`, \`raw_app.yaml\`, \`sql_to_apply/\`). That layout is only relevant for the terminal workflow — in chat, apps are addressed via the tool surface above. # Windmill raw app authoring reference @@ -2403,9 +2396,8 @@ function getResourceInstructions(): string { - A resource draft is a workspace item: \`{ type: 'resource', path, summary?, value, isDraft }\`. \`value\` is a CreateResource body: \`{ path, value, description?, resource_type, labels? }\` where the inner \`value\` is the resource type's data shape. - Reading a variable returns \`{ type: 'variable', path, summary?, isSecret, isDraft }\` — never its value, secret or not. \`isSecret\` tells you whether the value is encrypted. - \`write_variable\` takes \`{ path, value?, is_secret?, description?, account?, is_oauth?, expires_at?, labels? }\`. Creating a variable needs \`value\` and \`is_secret\`; editing one needs only the fields you are changing. Omitting \`value\` keeps the stored value, which is the only way to edit a secret variable — you cannot read its value, so passing any \`value\` you did not get from the user destroys it. -- For secret fields in a resource value, do NOT inline the raw secret. Create a Variable first with \`is_secret: true\`, then in the resource value reference it as \`"$var:path/to/variable"\`. +- For secret fields in a resource value, create the variable with \`write_variable\` and \`is_secret: true\`, and deploy it before the resource (see "Secrets" in the reference below). - Reference formats inside resource values: \`$var:g/all/name\` (global), \`$var:u/user/name\` (user), \`$var:f/folder/name\` (folder). Reference another resource with \`$res:path/to/resource\`. The same strings are also how a resource or variable is passed as a run argument (see the run-argument rule in the resource reference below); what they are never valid as is a variable's own value. -- When deploying drafts that depend on each other (e.g., a resource and the variables it references), deploy the variables first. - Use \`search_resource_types\` to discover valid \`resource_type\` names and their JSON Schemas. Match the resource value to that schema. - For OAuth resources, the \`is_oauth: true\` flag is managed by Windmill's OAuth flow; global mode generally creates manual resources, not OAuth ones. diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index c2f1a95d7a..e5767633bb 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -203,11 +203,11 @@ components: - height color: type: string - description: Color of the note (e.g., "yellow", "#ffff00") + description: "Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled." type: type: string enum: [free, group] - description: Type of note - 'free' for standalone notes, 'group' for notes that group other nodes + description: Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead. locked: type: boolean default: false @@ -245,7 +245,7 @@ components: description: ID of the last flow module in this group (topological exit point) color: type: string - description: Color for the group in the flow editor + description: "Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one." required: - start_id - end_id diff --git a/system_prompts/README.md b/system_prompts/README.md index 6dc8835a50..cc16473c6c 100644 --- a/system_prompts/README.md +++ b/system_prompts/README.md @@ -1,32 +1,104 @@ # System Prompts -This directory contains the single source of truth for AI system prompts used by both the frontend copilot and CLI guidance. +The single source of the AI guidance Windmill ships: what an agent needs to know about Windmill +itself to write flows, scripts, apps, pipelines, resources, triggers and schedules. Two consumers +are built from it: + +- **the chat** (the frontend AI chat: flow, script, app and global modes), through the + `$system_prompts` alias to `auto-generated/`; +- **the CLI** (`wmill init` / `wmill refresh prompts`), as the skills embedded in + `cli/src/guidance/skills.gen.ts`, next to the `AGENTS.wmill.md` template in + `cli/src/guidance/core.ts`. + +Guidance written in one consumer only is guidance the other never gets. Write it here. ## Structure ``` system_prompts/ -├── base/ # Core instruction templates (manually written) -│ ├── flow-base.md # Shared OpenFlow structure guidance -│ └── flow-cli.md # CLI/local-agent workflow guidance for write-flow skill -├── languages/ # Language-specific instructions (manually written) -└── auto-generated/ # Auto-generated files (DO NOT EDIT) - ├── sdks/ # SDK documentation - ├── cli/ # CLI command documentation - ├── prompts.ts # TypeScript exports - └── index.ts # Helper functions +├── base/ # Hand-written guidance, one file per topic (flow-base.md, raw-app.md, …) +│ # plus CLI-only intros (flow-cli.md, script-cli.md, raw-app-cli.md, …) +├── languages/ # Hand-written per-language guidance (bun.md, python3.md, …) +├── generate.py # Builds everything under auto-generated/ and cli/src/guidance/skills.gen.ts +├── utils.py # Shared helpers, including the fence renderer +└── auto-generated/ # Generated — never edit + ├── prompts.ts # One export per base/ and languages/ file (chat render), SDK docs, schema + ├── index.ts # Chat helpers (getFlowPrompt, getScriptPrompt, …) + ├── skills/ # One SKILL.md per CLI skill (cli render) + ├── sdks/ # SDK reference extracted from the TypeScript and Python clients + └── cli/ # CLI command reference extracted from cli/src/commands/ ``` -## Usage +## How a topic is assembled -### Regenerating Prompts +`TOPICS` in `generate.py` lists, for each topic, the parts both consumers concatenate in order: +a file under `base/` or `languages/`, or a token filled per consumer (`{lang}`, `{sdk}`, +`{wac_sdk}`, `{openflow_schema}`, `{cli_commands}`). The same entry produces the topic's chat +helper in `index.ts` and its CLI skill, so a part added there reaches both. A part tagged +`('chat', …)` or `('cli', …)` goes to that consumer only, and `cli_intro` heads the CLI skill with +a CLI-only workflow file (`flow-cli.md`, `script-cli.md`, `raw-app-cli.md`). -When SDK methods or the OpenFlow schema change, run: +| Topic | Chat helper | CLI skill | Shared parts | +|---|---|---|---| +| script | `getScriptPrompt(lang)` | `write-script-` | `script-base.md`, the language file, its SDK | +| flow | `getFlowPrompt()` | `write-flow` | `flow-base.md`, the OpenFlow schema | +| raw app | `getRawAppPrompt(lang)` | `raw-app` | `raw-app.md` (chat adds the SDK) | +| resources | `getResourcePrompt()` | `resources` | `resources.md` | +| workflow-as-code | `getWorkflowAsCodePrompt(lang)` | `write-workflow-as-code` | `workflow-as-code.md`, the WAC SDK | +| pipeline | `getPipelinePrompt()` | `write-pipeline` | `pipeline-base.md` | +| triggers, schedules, preview, CLI commands | — | same names | CLI only | + +The app chat embeds `RAW_APP_BASE` directly (`chat/app/core.ts`). + +## Consumer fences + +A sentence that only one consumer should see stays in the shared file, fenced: + +```md +A tool name outside that character set fails every run of the flow. + +The flow write tools refuse such a name. + + +`wmill lint ` reports it before anything runs. + +``` + +`render_for` (`utils.py`) renders every file once per consumer: `prompts.ts` gets the chat render, +the skills get the cli render, and the fence lines reach neither. Fences sit on their own lines and +cannot nest; an unbalanced or misspelled fence fails generation (an HTML comment opening with `cli` +or `chat`, or ending in `only`, is read as a fence attempt). + +Use a fence for a sentence or a section. When most of a topic differs per consumer, a CLI-only +file used as `cli_intro` reads better than a file that is mostly fences. + +## What goes where + +- **Here:** anything true of Windmill regardless of who is asking — module shapes, data flow + rules, what the editor accepts, how data tables, secrets or app access work. +- **In the chat's TypeScript prompt builders** (`frontend/src/lib/components/copilot/chat/*/core.ts`): + only tool plumbing (which tool to call, its arguments) and values known at run time (the user's + name, this app's data-table policy, session capabilities). +- **In `cli/src/guidance/core.ts`** (`AGENTS.wmill.md`): project-level CLI workflow — which skill + to use, deploying, debugging jobs. + +A file in `TOPICS` names no chat tool, not even inside a `chat-only` fence: it reaches every chat +mode (flow mode's system prompt, global mode's `get_instructions`), each with its own tool names, +and `global/sessionToolset.test.ts` fails a prompt that names a tool its session lacks. Describe the +action instead ("the flow write tools refuse such a name") and name the tool in TypeScript. The one +exception is `flow-chat-special-modules.md`, which only flow mode reads. + +## Regenerating + +After editing anything here, or when SDK methods, the OpenFlow schema or CLI commands change: ```bash python system_prompts/generate.py ``` +CI (`.github/workflows/check-system-prompts.yml`) runs `system_prompts/check-freshness.sh`, which +fails when the committed output is stale. + To also refresh the standalone skills in a Claude plugin checkout: ```bash @@ -52,53 +124,3 @@ suitable for ingestion by docs aggregators. In CI this runs from generator refuses to wipe the target directory unless it's empty or has a context7 marker (`context7.json`, `manifest.json`, or a `windmill-cli-docs` git remote), so a typo can't delete unrelated files. - -This will: - -1. Parse TypeScript and Python SDK files to extract function signatures -2. Parse the OpenFlow YAML schema -3. Parse the CLI commands -4. Assemble complete prompts from markdown files -5. Generate TypeScript exports in `auto-generated/` -6. Optionally refresh plugin-ready standalone `SKILL.md` files in the target directory - -### Scope - -These system prompts contain ONLY: - -- How to write Windmill scripts (language syntax, conventions, SDK usage) -- How to structure Windmill flows (OpenFlow schema, module types, data flow) -- Resource type handling, S3 operations - -They DO NOT contain: - -- Tool usage instructions (edit_code, set_flow_json, etc.) -- IDE/editor specific commands -- Testing tool invocations - -Tool instructions are added separately by the frontend and CLI. - -CLI-only workflow instructions live in `base/flow-cli.md` and are included in the -generated `write-flow` skill for `wmill init`. They are intentionally excluded -from the frontend flow chat prompt. - -## Integration - -### Frontend - -Uses Vite path alias `$system_prompts` pointing to `auto-generated/`: - -```typescript -import { FLOW_GUIDANCE } from "$system_prompts/flow"; -import { getLangContext } from "$system_prompts/languages"; -``` - -### CLI - -Generates `/cli/src/guidance/skills.gen.ts` with embedded skill content for `wmill init`. - -## Editing Guidelines - -- Edit markdown files in `base/`, `languages/` -- Never edit files in `auto-generated/` directly -- After editing, run `generate.py` to update exports diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index b18638ee23..5f71cb9625 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -375,7 +375,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file +Validate Windmill flow, schedule, and trigger YAML files in a directory (including AI agent tool names and flow groups/notes), and report script metadata that has no deployable content file **Arguments:** `[directory:string]` diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index 9cf32d7bd1..6efdf6b64a 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -170,8 +170,8 @@ names, so neither is name-checked at all — leave those summaries as they are. - Always set `summary`. It must be unique among that agent's tools, and must not be one of the reserved ids (`do`, `bg`, `ctx`, `state`, `if`, `else`, `for`, `delete`, `while`, `new`, `in`, `failure`, `preprocessor`, `as`, `Input`, `Result`, `Trigger`) -- A tool name outside that character set is rejected: flow write tools refuse it, and a flow that - reaches the worker with one fails every run with `Invalid tool name` +- A tool name outside that character set fails any run that offers the tool to the agent, with `Invalid tool name`. + The flow write tools refuse such a name. - Tool `id` follows the same rules as any module ID — unique across the flow, underscores not spaces - `description` is optional free text telling the agent when and how to call the tool. Set it whenever the name alone does not make that obvious; it overrides the description derived from the @@ -194,9 +194,11 @@ names, so neither is name-checked at all — leave those summaries as they are. ## Loop Structure Rules +- A `forloopflow` runs its `modules` once per element of `iterator`, a javascript expression returning an array (e.g. `results.get_items`); `parallel: true` runs the iterations concurrently, and `skip_failures: true` carries on past a failed iteration - For `whileloopflow`, break the loop with a module-level `stop_after_if`: on the loop module itself, or on an inner step (required when that step carries state via its own `results` — see below) - `stop_after_if` is always a sibling of `id` and `value` on a flow module — never a direct key of the loop's `value` object - `stop_after_all_iters_if` is for checks after the whole loop finishes, not the normal per-iteration break condition +- `stop_after_if` is evaluated after each iteration: on the loop module, `result` is that iteration's result (what its last step returned); on an inner step, it is that step's result - `flow_input.iter.value` in a `whileloopflow` is just the iteration index (same number as `flow_input.iter.index`) — it never carries state, so `flow_input.iter.value.` is always undefined and a loop whose stop condition depends on it never terminates - To carry state across iterations, a step reads its own previous-iteration result via `results.` with a first-iteration fallback (e.g. `results.b ?? flow_input.start`) — but then the loop's `stop_after_if` MUST sit on that inner step, not on the loop module: a body that is exactly one plain step with the stop condition on the loop module runs on a fast path where `results.` is null on every iteration and the loop never terminates (bodies with 2+ steps, or whose single step has its own `stop_after_if`, retry or similar, resolve `results` across iterations regardless of stop placement) - For state that is just a counter, derive it from the index instead (e.g. `flow_input.iter.index + 1`) — that works in every configuration, including with `stop_after_if` on the loop module @@ -334,6 +336,7 @@ Incorrect shape (identity has no resume URLs — not a real approval): ## Branch Result Scope Rules +- A `branchone` runs the first of its `branches` whose `expr` is true, in order, and its `default` modules when none is; a `branchall` runs every branch (concurrently with `parallel: true`) - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch - Outside a `branchone`, do NOT reference ids of steps that only exist inside its branches or default branch. Use `results.` instead - Outside a `branchall`, do NOT reference ids of steps inside its branches. Use `results.` instead @@ -393,6 +396,40 @@ JavaScript transform (dynamic expression): - For flow inputs: Use type `"object"` with format `"resource-{type}"` (e.g., `"resource-postgresql"`) - For step inputs: Use static value `"$res:path/to/resource"` +## Reusing Existing Scripts and Flows + +Unless the user asked for new code, look for a workspace script or flow that already does a step's job before writing it, and reuse it by path instead of copying its logic into a rawscript: + +- a workspace script: `type: script` with `path` (e.g. `f/folder/send_email`) +- a workspace flow, run as a subflow: `type: flow` with `path` +- a Hub script: `type: script` with a `hub///` path + +The step's `input_transforms` must cover the reused item's inputs, so read its input schema first. + +## Organizing Flows: Groups and Notes + +Groups and notes shape how a flow reads in the editor; neither changes what it does. + +**Segment every non-trivial flow into groups without waiting to be asked.** Whenever a flow has more than a couple of steps, or consecutive steps form a stage ("fetch", "transform", "notify"), put them in a group, and aim for every meaningful step to belong to one. Use notes sparingly, for flow-wide information that belongs to no span of steps: the flow's purpose, key assumptions, warnings, TODOs. One note is usually enough; never label a run of steps with a note, which is what a group is for. + +`value.groups` lists the groups, each spanning the steps from `start_id` to `end_id`: + +- `start_id`, `end_id` (required): ids of the group's first and last step; the same id for both makes a one-step group +- `summary`: the group's title +- `note`: markdown shown under the title +- `color`: one of `yellow`, `blue`, `green`, `purple`, `pink`, `orange`, `red`, `cyan`, `lime`, `gray`, never a hex code or CSS color; leave it out and the editor picks one +- `autocollapse`: `true` shows the group collapsed by default + +The editor refuses to draw a flow whose groups break any of these rules: + +- `start_id` and `end_id` are steps of the same list: both top-level, or both in the same loop body or branch. A group can hold a loop or branch step whole, but cannot start outside one and end inside it +- `start_id` does not come after `end_id` in that list +- groups nest (one entirely inside another) but never partly overlap, and no two groups share both `start_id` and `end_id` +- groups hold ordinary steps only: never `preprocessor`, `failure`, `Input`, `Result`, `Trigger`, or an AI agent's tools + +`value.notes` lists sticky notes, each with a unique `id`, markdown `text`, a `color` from the same list, and `type: free`. The `group` note type is deprecated; use `value.groups` instead. +Leave a note's `position` and `size` out and they are filled in for you. + ## Final Structural Self-Check Before finalizing a flow, verify: @@ -402,6 +439,7 @@ Before finalizing a flow, verify: - any approval step has module-level `suspend` - no downstream step references inner branch step ids from outside the branch - every AI agent flowmodule tool has a unique `summary` made only of letters, numbers and underscores +- every group starts and ends on steps of the same list, start before end, nesting without partial overlap ## S3 Object Operations @@ -459,4 +497,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/index.ts b/system_prompts/auto-generated/index.ts index 1d28c047b5..ade95efe9e 100644 --- a/system_prompts/auto-generated/index.ts +++ b/system_prompts/auto-generated/index.ts @@ -46,7 +46,9 @@ export function getFlowPrompt(): string { // Helper for resource & variable authoring export function getResourcePrompt(): string { - return prompts.RESOURCES_BASE; + return [ + prompts.RESOURCES_BASE + ].filter(Boolean).join('\n\n'); } // Helper for raw app authoring (chat consumers). Inline backend runnables are @@ -66,7 +68,9 @@ export function getRawAppPrompt(language?: string): string { // Helper for data pipeline authoring (chat consumers) export function getPipelinePrompt(): string { - return prompts.PIPELINE_BASE; + return [ + prompts.PIPELINE_BASE + ].filter(Boolean).join('\n\n'); } // Helper to get the datatable SQL SDK reference (wmill.datatable()). diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index ba5f382651..77b75a41e6 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -4,29 +4,24 @@ export const SCRIPT_BASE = `# Windmill Script Writing Guide ## General Principles -- Scripts must export a main function (do not call it) +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters - Libraries are installed automatically - do not show installation instructions -- Credentials and configuration are stored in resources and passed as parameters -- The windmill client (\`wmill\`) provides APIs for interacting with the platform - -## Function Naming - -- Main function: \`main\` (or \`preprocessor\` for preprocessor scripts) -- Must be async for TypeScript variants +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it \`main\` (\`Main\` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no \`main\`: their language section shows how they take arguments +- Where the language has a Windmill client (\`wmill\`), use it to interact with the platform ## Return Values -- Scripts can return any JSON-serializable value +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces - Return values become available to subsequent flow steps via \`results.step_id\` ## Preprocessor Scripts -Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named \`preprocessor\` instead of \`main\`, and it receives a single parameter called \`event\` (the language section gives its type). The returned object determines the parameter values passed to the flow. e.g., \`{ b: 1, a: 2 }\` calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. - -The preprocessor receives a single parameter called \`event\`. `; export const FLOW_BASE = `# Windmill Flow Building Guide @@ -201,8 +196,8 @@ names, so neither is name-checked at all — leave those summaries as they are. - Always set \`summary\`. It must be unique among that agent's tools, and must not be one of the reserved ids (\`do\`, \`bg\`, \`ctx\`, \`state\`, \`if\`, \`else\`, \`for\`, \`delete\`, \`while\`, \`new\`, \`in\`, \`failure\`, \`preprocessor\`, \`as\`, \`Input\`, \`Result\`, \`Trigger\`) -- A tool name outside that character set is rejected: flow write tools refuse it, and a flow that - reaches the worker with one fails every run with \`Invalid tool name\` +- A tool name outside that character set fails any run that offers the tool to the agent, with \`Invalid tool name\`. + The flow write tools refuse such a name. - Tool \`id\` follows the same rules as any module ID — unique across the flow, underscores not spaces - \`description\` is optional free text telling the agent when and how to call the tool. Set it whenever the name alone does not make that obvious; it overrides the description derived from the @@ -225,9 +220,11 @@ names, so neither is name-checked at all — leave those summaries as they are. ## Loop Structure Rules +- A \`forloopflow\` runs its \`modules\` once per element of \`iterator\`, a javascript expression returning an array (e.g. \`results.get_items\`); \`parallel: true\` runs the iterations concurrently, and \`skip_failures: true\` carries on past a failed iteration - For \`whileloopflow\`, break the loop with a module-level \`stop_after_if\`: on the loop module itself, or on an inner step (required when that step carries state via its own \`results\` — see below) - \`stop_after_if\` is always a sibling of \`id\` and \`value\` on a flow module — never a direct key of the loop's \`value\` object - \`stop_after_all_iters_if\` is for checks after the whole loop finishes, not the normal per-iteration break condition +- \`stop_after_if\` is evaluated after each iteration: on the loop module, \`result\` is that iteration's result (what its last step returned); on an inner step, it is that step's result - \`flow_input.iter.value\` in a \`whileloopflow\` is just the iteration index (same number as \`flow_input.iter.index\`) — it never carries state, so \`flow_input.iter.value.\` is always undefined and a loop whose stop condition depends on it never terminates - To carry state across iterations, a step reads its own previous-iteration result via \`results.\` with a first-iteration fallback (e.g. \`results.b ?? flow_input.start\`) — but then the loop's \`stop_after_if\` MUST sit on that inner step, not on the loop module: a body that is exactly one plain step with the stop condition on the loop module runs on a fast path where \`results.\` is null on every iteration and the loop never terminates (bodies with 2+ steps, or whose single step has its own \`stop_after_if\`, retry or similar, resolve \`results\` across iterations regardless of stop placement) - For state that is just a counter, derive it from the index instead (e.g. \`flow_input.iter.index + 1\`) — that works in every configuration, including with \`stop_after_if\` on the loop module @@ -365,6 +362,7 @@ Incorrect shape (identity has no resume URLs — not a real approval): ## Branch Result Scope Rules +- A \`branchone\` runs the first of its \`branches\` whose \`expr\` is true, in order, and its \`default\` modules when none is; a \`branchall\` runs every branch (concurrently with \`parallel: true\`) - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch - Outside a \`branchone\`, do NOT reference ids of steps that only exist inside its branches or default branch. Use \`results.\` instead - Outside a \`branchall\`, do NOT reference ids of steps inside its branches. Use \`results.\` instead @@ -424,6 +422,40 @@ JavaScript transform (dynamic expression): - For flow inputs: Use type \`"object"\` with format \`"resource-{type}"\` (e.g., \`"resource-postgresql"\`) - For step inputs: Use static value \`"$res:path/to/resource"\` +## Reusing Existing Scripts and Flows + +Unless the user asked for new code, look for a workspace script or flow that already does a step's job before writing it, and reuse it by path instead of copying its logic into a rawscript: + +- a workspace script: \`type: script\` with \`path\` (e.g. \`f/folder/send_email\`) +- a workspace flow, run as a subflow: \`type: flow\` with \`path\` +- a Hub script: \`type: script\` with a \`hub///\` path + +The step's \`input_transforms\` must cover the reused item's inputs, so read its input schema first. + +## Organizing Flows: Groups and Notes + +Groups and notes shape how a flow reads in the editor; neither changes what it does. + +**Segment every non-trivial flow into groups without waiting to be asked.** Whenever a flow has more than a couple of steps, or consecutive steps form a stage ("fetch", "transform", "notify"), put them in a group, and aim for every meaningful step to belong to one. Use notes sparingly, for flow-wide information that belongs to no span of steps: the flow's purpose, key assumptions, warnings, TODOs. One note is usually enough; never label a run of steps with a note, which is what a group is for. + +\`value.groups\` lists the groups, each spanning the steps from \`start_id\` to \`end_id\`: + +- \`start_id\`, \`end_id\` (required): ids of the group's first and last step; the same id for both makes a one-step group +- \`summary\`: the group's title +- \`note\`: markdown shown under the title +- \`color\`: one of \`yellow\`, \`blue\`, \`green\`, \`purple\`, \`pink\`, \`orange\`, \`red\`, \`cyan\`, \`lime\`, \`gray\`, never a hex code or CSS color; leave it out and the editor picks one +- \`autocollapse\`: \`true\` shows the group collapsed by default + +The editor refuses to draw a flow whose groups break any of these rules: + +- \`start_id\` and \`end_id\` are steps of the same list: both top-level, or both in the same loop body or branch. A group can hold a loop or branch step whole, but cannot start outside one and end inside it +- \`start_id\` does not come after \`end_id\` in that list +- groups nest (one entirely inside another) but never partly overlap, and no two groups share both \`start_id\` and \`end_id\` +- groups hold ordinary steps only: never \`preprocessor\`, \`failure\`, \`Input\`, \`Result\`, \`Trigger\`, or an AI agent's tools + +\`value.notes\` lists sticky notes, each with a unique \`id\`, markdown \`text\`, a \`color\` from the same list, and \`type: free\`. The \`group\` note type is deprecated; use \`value.groups\` instead. +Leave a note's \`position\` and \`size\` out and they are filled in for you. + ## Final Structural Self-Check Before finalizing a flow, verify: @@ -433,6 +465,7 @@ Before finalizing a flow, verify: - any approval step has module-level \`suspend\` - no downstream step references inner branch step ids from outside the branch - every AI agent flowmodule tool has a unique \`summary\` made only of letters, numbers and underscores +- every group starts and ends on steps of the same list, start before end, nesting without partial overlap ## S3 Object Operations @@ -492,12 +525,6 @@ export const RESOURCES_BASE = `# Windmill Resources Resources store credentials and configuration for external services. -## File Format - -Resource files use the pattern: \`{path}.resource.json\` - -Example: \`f/databases/postgres_prod.resource.json\` - ## Resource Structure \`\`\`json @@ -537,6 +564,13 @@ Reference variables in resource values: - \`$var:u/username/name\` - User variable - \`$var:f/folder/name\` - Folder variable +## Secrets + +Never put a secret (password, API key, token) inline in a resource value. Store it in a secret variable and reference that variable as \`$var:\`. + +- \`$var:\` is a reference, not a value: it resolves to the variable's value at run time. Never invent a value for a variable. +- A resource that references a variable needs the variable to exist first, so create or deploy the variable before the resource. + ## Resource References Reference other resources: @@ -750,23 +784,6 @@ def main(db: postgresql): pass \`\`\` -## CLI Commands - -\`\`\`bash -# List resources -wmill resource list - -# List resource types with schemas -wmill resource-type list --schema - -# Get specific resource type schema -wmill resource-type get postgresql - -# Deploy resources to the workspace — destructive to remote state, so only run when -# the user explicitly asks to deploy/publish/push. Depending on how the repo is wired, -# deploy via \`git push\` or \`wmill sync push\` (see the Deploying section in AGENTS.wmill.md). -wmill sync push -\`\`\` `; export const RAW_APP_BASE = `# Windmill Raw Apps @@ -819,6 +836,8 @@ Import the generated bindings and call the runnable like a function. \`./wmill\` | \`getJob(jobId)\` | a \`Job\` (\`{ type, success, result, duration_ms, ... }\`) | polling status without blocking | | \`streamJob(jobId, onUpdate?)\` | the final result, calling \`onUpdate\` per chunk | showing output as it is produced | +A runnable is always called with **one object** whose keys are its \`main\` parameters — \`main(user_id: string, limit: number)\` is called as \`backend.get_users({ user_id, limit })\`, never with positional arguments. A runnable without parameters is called with no argument. Resource and variable ids handed to the \`wmill\` client are paths (\`u//\` or \`f//\`). + Run and wait — the common case: \`\`\`tsx @@ -892,9 +911,9 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo ### Inline runnables -Inline runnables carry their own source code. For file-based raw apps, the runnable language is determined by the backend file extension. The script must expose a \`main\` function as its entrypoint. +Inline runnables carry their own source code, and must expose a \`main\` function as their entrypoint. -**TypeScript example** (\`backend/get_user.ts\`): +**TypeScript example** (runnable \`get_user\`): \`\`\`typescript import * as wmill from 'windmill-client'; @@ -906,7 +925,7 @@ export async function main(user_id: string) { } \`\`\` -**Python example** (\`backend/get_user.py\`): +**Python example** (runnable \`get_user\`): \`\`\`python import wmill @@ -923,12 +942,14 @@ An inline runnable runs as an ordinary Windmill job. \`import * as wmill from 'w **Don't read \`WM_TOKEN\` or \`BASE_INTERNAL_URL\` and build an API URL to \`fetch\`.** The client's own \`setClient\` already reads exactly those, and it also sets the credentials mode a raw app needs (\`WM_RAW_APP\` suppresses credentials, because a sandboxed bundle calls the API from an opaque origin that can never pair with \`Access-Control-Allow-Origin: *\`). Rebuilding that by hand drops the parts you can't see. Use \`wmill.*\` for everything Windmill, and \`fetch\` only for third-party APIs. -Prefer the \`wmill\` functions that appear in the SDK reference; for an endpoint none of them covers, the generated service classes (\`JobService\`, \`ScriptService\`, ...) are importable from \`windmill-client\`. What is not available is a name you guessed at: \`getBaseUrl\` and \`getWorkspaceToken\` are inventions, not API. +Use only \`wmill\` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (\`JobService\`, \`ScriptService\`, ...) are importable from \`windmill-client\`. What is not available is a name you guessed at: \`getBaseUrl\` and \`getWorkspaceToken\` are inventions, not API. ### Path runnables (script / flow / hubscript) When \`type\` is \`script\`, \`flow\`, or \`hubscript\`, the runnable just stores a \`path\` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at \`hub///\`) for a third-party service, and reference it instead of copying its logic. + ### Draft code vs deployed code This decides whether an app works before anything is deployed: @@ -948,15 +969,27 @@ Prefer a **path runnable of type \`flow\`** over an inline runnable that calls \ \`staticInputs\` is an optional \`Record\` for arguments not overridable from the frontend. Useful with path runnables to pre-fill some args while leaving the rest to the frontend caller. +## Who can open a deployed app + +A draft app is reachable by nobody; deploying is what exposes it, and its backend runnables with it. Otherwise only users with access to the app can open it, unless the app is opened to: + +- **anonymous** users: anyone with the URL, without logging in; +- **guests**: anyone the instance's identity provider authenticates, whether or not they belong to the workspace. + +Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. +When a deploy widens who may open the app, tell the user in plain words. + ## Data Tables -Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. +Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. ### Critical rules -1. **Whitelisted tables only**: a runnable can only query tables listed in the app's \`data.tables\` config. Tables not in this list are not accessible. -2. **Add tables before using**: queries against unlisted tables fail at runtime. When you introduce a new table, register it in \`data.tables\` first. -3. **Use the configured datatable/schema**: the app's \`data\` config sets the default datatable and schema; reference them consistently across runnables. +1. **Check what exists first**: look up the workspace's data tables and their tables before designing storage, and reuse a suitable table rather than creating another. Never assume a \`main\` data table exists. +2. **Whitelisted tables only**: a runnable can only query tables listed in the app's \`data.tables\` config. Queries against unlisted tables fail at runtime, so register a new table there before using it. +3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. +4. **Qualify table names**: an unqualified name means the \`public\` schema, so write every other table as \`schema.table\`, in table creation and queries alike. The app's \`data\` config sets the default datatable and schema its tables go in; use them consistently across runnables. +5. **Pass the role**: when the app's \`data.roles\` gives a data table a role, every \`wmill.datatable\` call on it passes that role (\`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python). The role only reaches what it was granted, so a query outside it fails with \`permission denied\`. ### Querying in TypeScript (Bun/Deno) @@ -1026,7 +1059,8 @@ Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lak ## Storage prerequisites -A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check with the \`list_ducklakes\` tool before you build (it returns the configured DuckLake catalogs, or none). Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if \`list_ducklakes\` returns none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: +A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. +Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if there is none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: - a workspace **admin** sets it up in Workspace settings → Object Storage (add an S3/Azure/GCS storage), then adds a DuckLake catalog on top of it; - anyone **without admin rights** should ask a workspace admin to configure object storage + a DuckLake catalog. @@ -1050,7 +1084,7 @@ A script joins the pipeline when its source begins with the \`pipeline\` annotat SELECT * FROM read_csv($file) \`\`\` - **Outputs** are inferred from what the body writes — a \`CREATE TABLE\`, a \`wmill.writeS3File(...)\`, a DuckLake/datatable write. To declare a managed output explicitly, use \`// materialize \`. -- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` one). +- Optional badges: \`// partitioned \`, \`// freshness \` (e.g. \`1h\`), \`// tag \`, \`// retry [delay]\`, \`// data_test ...\` (managed DuckLake targets only — deploy rejects it beside a \`dbt://\` one), \`// measure = [where ]\` and \`// dimension = \` (see "Declared metrics" below). ## S3 object wiring (storage form matters) @@ -1077,19 +1111,30 @@ A managed \`// materialize ducklake:///
    \` tells the runtime to writ \`// materialize manual \` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. -\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. +\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table). The \`{partition}\` token, usable in any asset URI **and** in the body SQL, is replaced at run time by the current partition's **identity string**: + +- To filter the source to the active slice on a time grain, use the runtime-injected macro: \`WHERE wm_partition() = {partition}\`. \`wm_partition(ts)\` buckets a timestamp in exactly the identity format the runtime uses for daily/hourly/weekly/monthly, so it always matches; never hand-write a \`strftime\` format. +- Do NOT write \`= TIMESTAMP {partition}\`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at run time. +- For \`dynamic\` partitioning the identity is the caller-supplied key (not a timestamp, no macro), so filter on it directly: \`WHERE = {partition}\`. \`materialize\` is an output **declaration** on a node — not a command. There is no "materialize run". -## How to build one in chat +## Declared metrics (\`measure\` / \`dimension\`) + +On a node that materializes a DuckLake table, \`// measure = [where ]\` names the canonical way to aggregate that table (e.g. \`// measure revenue = sum(amount) where not is_refund\`), and \`// dimension = \` names a way to slice it (e.g. \`// dimension region = region\`, \`// dimension month = date_trunc('month', ordered_at)\`). They execute nothing: they are catalogued at deploy so the editor and other agents reuse the definition instead of re-deriving it and silently disagreeing. + +- Keep the predicate in the \`where\` clause rather than folding it into the aggregate: it is rendered as \` FILTER (WHERE )\`, which is what lets two measures with different predicates share one GROUP BY. +- DuckLake-only, and only meaningful next to \`// materialize\`. +- Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures — an obvious \`count(*)\` earns nothing. +- To use a metric another node declares, read that node and reuse its exact expression rather than guessing it. + +## How to build one 1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. -2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`). Default to \`duckdb\` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick \`postgresql\`, \`bun\`, or \`python3\` only when that section says the work calls for it. +2. Write each node as its own script. Default to \`duckdb\` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick \`postgresql\`, \`bun\`, or \`python3\` only when that section says the work calls for it. 3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. 4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. -5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. - -When the user already has the \`/pipeline/\` editor open, prefer the dedicated \`build_pipeline_node\` / \`edit_pipeline_node\` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. +5. Don't deploy nodes unless the user asks to. A pipeline only "runs" once its scripts are deployed and their triggers exist. ## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) @@ -3297,7 +3342,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\`: it is ignored when memory is off, and an older \`auto\` or\\n\`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\`: it is ignored when memory is off, and an older \`auto\` or\\n\`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands @@ -3676,7 +3721,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file +Validate Windmill flow, schedule, and trigger YAML files in a directory (including AI agent tool names and flow groups/notes), and report script metadata that has no deployable content file **Arguments:** \`[directory:string]\` @@ -4367,8 +4412,6 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. -Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. - ## Imports \`\`\`typescript @@ -4390,7 +4433,7 @@ import * as wmill from "windmill-client"; **Prefer \`windmill-client\` over raw \`fetch\` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve \`fetch\` for calling *external* HTTP APIs that aren't Windmill. -The full \`windmill-client\` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method to use instead of guessing or falling back to \`fetch\`. +The full \`windmill-client\` API reference (every exported function and its signature) is included below — consult it for the exact method to use instead of guessing or falling back to \`fetch\`. ## Preprocessor Scripts @@ -4408,7 +4451,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -4490,8 +4535,6 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. -Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. - ## Imports **The constraint is the runtime, not the import list.** You may import npm packages and relative Windmill scripts; they are resolved and bundled exactly like a regular Bun script. But the native worker only provides \`fetch\` and the JavaScript standard library, so any imported code must work using only those. Anything requiring Node/Bun built-ins (\`node:fs\`, \`child_process\`, the \`Bun\` API, native modules) belongs in a regular \`bun\` script instead. Use the globally available \`fetch\` for HTTP: @@ -4508,7 +4551,7 @@ export async function main(url: string) { \`windmill-client\` works on the native worker (its calls go over \`fetch\`), so use it as the **preferred way to talk to Windmill** — reading resources/variables/states, running scripts and flows, and the S3 helpers below (\`loadS3File\`, \`loadS3FileStream\`, \`writeS3File\`, \`S3Object\`). It handles auth, the workspace, and the base URL for you. Reserve raw \`fetch\` for calling *external* HTTP APIs that aren't Windmill. -The full \`windmill-client\` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of hand-rolling a \`fetch\` against the Windmill API. +The full \`windmill-client\` API reference (every exported function and its signature) is included below — consult it for the exact method instead of hand-rolling a \`fetch\` against the Windmill API. ## Preprocessor Scripts @@ -4527,7 +4570,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -4654,8 +4699,6 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. -Before using a resource type, check the \`rt.d.ts\` file in the project root to see all available resource types and their fields. This file is generated by \`wmill resource-type generate-namespace\`. - ## Imports \`\`\`typescript @@ -4677,7 +4720,7 @@ import * as wmill from "windmill-client"; **Prefer \`windmill-client\` over raw \`fetch\` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve \`fetch\` for calling *external* HTTP APIs that aren't Windmill. -The full \`windmill-client\` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of guessing or falling back to \`fetch\`. +The full \`windmill-client\` API reference (every exported function and its signature) is included below — consult it for the exact method instead of guessing or falling back to \`fetch\`. ## Preprocessor Scripts @@ -4695,7 +4738,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -5296,7 +5341,7 @@ For preprocessor scripts, the function should be named \`preprocessor\` and rece from typing import TypedDict, Literal, Any class Event(TypedDict): - kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "amqp", "gcp", "azure"] body: Any headers: dict[str, str] query: dict[str, str] diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index afc064df40..a2feeaf166 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -2,30 +2,25 @@ ## General Principles -- Scripts must export a main function (do not call it) +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters - Libraries are installed automatically - do not show installation instructions -- Credentials and configuration are stored in resources and passed as parameters -- The windmill client (`wmill`) provides APIs for interacting with the platform - -## Function Naming - -- Main function: `main` (or `preprocessor` for preprocessor scripts) -- Must be async for TypeScript variants +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform ## Return Values -- Scripts can return any JSON-serializable value +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces - Return values become available to subsequent flow steps via `results.step_id` ## Preprocessor Scripts -Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). The returned object determines the parameter values passed to the flow. e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. -The preprocessor receives a single parameter called `event`. - # Ansible @@ -244,8 +239,6 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. -Before using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`. - ## Imports ```typescript @@ -267,7 +260,7 @@ import * as wmill from "windmill-client"; **Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method to use instead of guessing or falling back to `fetch`. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method to use instead of guessing or falling back to `fetch`. ## Preprocessor Scripts @@ -285,7 +278,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -367,8 +362,6 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. -Before using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`. - ## Imports **The constraint is the runtime, not the import list.** You may import npm packages and relative Windmill scripts; they are resolved and bundled exactly like a regular Bun script. But the native worker only provides `fetch` and the JavaScript standard library, so any imported code must work using only those. Anything requiring Node/Bun built-ins (`node:fs`, `child_process`, the `Bun` API, native modules) belongs in a regular `bun` script instead. Use the globally available `fetch` for HTTP: @@ -385,7 +378,7 @@ export async function main(url: string) { `windmill-client` works on the native worker (its calls go over `fetch`), so use it as the **preferred way to talk to Windmill** — reading resources/variables/states, running scripts and flows, and the S3 helpers below (`loadS3File`, `loadS3FileStream`, `writeS3File`, `S3Object`). It handles auth, the workspace, and the base URL for you. Reserve raw `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API. ## Preprocessor Scripts @@ -404,7 +397,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -531,8 +526,6 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. -Before using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`. - ## Imports ```typescript @@ -554,7 +547,7 @@ import * as wmill from "windmill-client"; **Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of guessing or falling back to `fetch`. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method instead of guessing or falling back to `fetch`. ## Preprocessor Scripts @@ -572,7 +565,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -1173,7 +1168,7 @@ For preprocessor scripts, the function should be named `preprocessor` and receiv from typing import TypedDict, Literal, Any class Event(TypedDict): - kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "amqp", "gcp", "azure"] body: Any headers: dict[str, str] query: dict[str, str] diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 3aafa0be85..2601a62380 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -380,7 +380,7 @@ Manage jobs (import/export) ### lint -Validate Windmill flow, schedule, and trigger YAML files in a directory, and report script metadata that has no deployable content file +Validate Windmill flow, schedule, and trigger YAML files in a directory (including AI agent tool names and flow groups/notes), and report script metadata that has no deployable content file **Arguments:** `[directory:string]` diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index 8c6b8e09da..fe9f57c55d 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -168,7 +168,7 @@ The `data` block in `raw_app.yaml` controls which tables the app can query. ```yaml data: datatable: main # Default datatable - schema: app_schema # Default schema (optional) + schema: app_schema # Schema the app's tables go in (optional); still write them as app_schema.
    tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema @@ -213,6 +213,8 @@ data: - main/users ``` +A migration runs with no default schema, so a table outside `public` is created with its schema, `CREATE TABLE IF NOT EXISTS app_schema.items (...)`, listed as `main/app_schema:items`, and queried as `app_schema.items`. + ### Migration best practices - **Use idempotent SQL**: `CREATE TABLE IF NOT EXISTS`, etc. @@ -222,8 +224,9 @@ data: ## CLI Commands -Two commands you run yourself, not the user: +Commands you run yourself, not the user: - `wmill app new` — run it with flags, per the "Creating a Raw App" section above. +- `wmill app lint ` — checks the app's structure and that it builds. Run it after editing, before offering a preview or a deploy; a bundle that compiles still says nothing about behavior, so a preview is what checks that. - `wmill generate-metadata` — (re)generates local lock files and refreshes `wmill-lock.yaml` content hashes; writes local files only (not a deploy). After adding or editing a runnable, offer it and run it on agreement — or automatically if the project's `AGENTS.md` opts into that (see "After creating a runnable" above). For the rest, tell the user which command fits their intent and let them run it — these deploy to the workspace, overwrite local files, or launch a long-running server, so the user should consent each time: @@ -235,8 +238,6 @@ For the rest, tell the user which command fits their intent and let them run it | `wmill sync push` | Deploy app to Windmill | | `wmill sync pull` | Pull latest from Windmill | - - # Windmill Raw Apps Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. @@ -287,6 +288,8 @@ Import the generated bindings and call the runnable like a function. `./wmill` i | `getJob(jobId)` | a `Job` (`{ type, success, result, duration_ms, ... }`) | polling status without blocking | | `streamJob(jobId, onUpdate?)` | the final result, calling `onUpdate` per chunk | showing output as it is produced | +A runnable is always called with **one object** whose keys are its `main` parameters — `main(user_id: string, limit: number)` is called as `backend.get_users({ user_id, limit })`, never with positional arguments. A runnable without parameters is called with no argument. Resource and variable ids handed to the `wmill` client are paths (`u//` or `f//`). + Run and wait — the common case: ```tsx @@ -360,9 +363,10 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo ### Inline runnables -Inline runnables carry their own source code. For file-based raw apps, the runnable language is determined by the backend file extension. The script must expose a `main` function as its entrypoint. +Inline runnables carry their own source code, and must expose a `main` function as their entrypoint. +On disk, a runnable's language is determined by its backend file extension. -**TypeScript example** (`backend/get_user.ts`): +**TypeScript example** (runnable `get_user`): ```typescript import * as wmill from 'windmill-client'; @@ -374,7 +378,7 @@ export async function main(user_id: string) { } ``` -**Python example** (`backend/get_user.py`): +**Python example** (runnable `get_user`): ```python import wmill @@ -391,12 +395,14 @@ An inline runnable runs as an ordinary Windmill job. `import * as wmill from 'wi **Don't read `WM_TOKEN` or `BASE_INTERNAL_URL` and build an API URL to `fetch`.** The client's own `setClient` already reads exactly those, and it also sets the credentials mode a raw app needs (`WM_RAW_APP` suppresses credentials, because a sandboxed bundle calls the API from an opaque origin that can never pair with `Access-Control-Allow-Origin: *`). Rebuilding that by hand drops the parts you can't see. Use `wmill.*` for everything Windmill, and `fetch` only for third-party APIs. -Prefer the `wmill` functions that appear in the SDK reference; for an endpoint none of them covers, the generated service classes (`JobService`, `ScriptService`, ...) are importable from `windmill-client`. What is not available is a name you guessed at: `getBaseUrl` and `getWorkspaceToken` are inventions, not API. +Use only `wmill` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (`JobService`, `ScriptService`, ...) are importable from `windmill-client`. What is not available is a name you guessed at: `getBaseUrl` and `getWorkspaceToken` are inventions, not API. ### Path runnables (script / flow / hubscript) When `type` is `script`, `flow`, or `hubscript`, the runnable just stores a `path` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at `hub///`) for a third-party service, and reference it instead of copying its logic. + ### Draft code vs deployed code This decides whether an app works before anything is deployed: @@ -416,15 +422,29 @@ Prefer a **path runnable of type `flow`** over an inline runnable that calls `wm `staticInputs` is an optional `Record` for arguments not overridable from the frontend. Useful with path runnables to pre-fill some args while leaving the rest to the frontend caller. +## Who can open a deployed app + +A draft app is reachable by nobody; deploying is what exposes it, and its backend runnables with it. Otherwise only users with access to the app can open it, unless the app is opened to: + +- **anonymous** users: anyone with the URL, without logging in; +- **guests**: anyone the instance's identity provider authenticates, whether or not they belong to the workspace. + +Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. +`public: true` in `raw_app.yaml` deploys the app for anonymous users, and `guests: true` for guests. Remove the line and the next push closes the app again. + ## Data Tables -Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. +Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. ### Critical rules -1. **Whitelisted tables only**: a runnable can only query tables listed in the app's `data.tables` config. Tables not in this list are not accessible. -2. **Add tables before using**: queries against unlisted tables fail at runtime. When you introduce a new table, register it in `data.tables` first. -3. **Use the configured datatable/schema**: the app's `data` config sets the default datatable and schema; reference them consistently across runnables. +1. **Check what exists first**: look up the workspace's data tables and their tables before designing storage, and reuse a suitable table rather than creating another. Never assume a `main` data table exists. +2. **Whitelisted tables only**: a runnable can only query tables listed in the app's `data.tables` config. Queries against unlisted tables fail at runtime, so register a new table there before using it. +3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. +4. **Qualify table names**: an unqualified name means the `public` schema, so write every other table as `schema.table`, in table creation and queries alike. The app's `data` config sets the default datatable and schema its tables go in; use them consistently across runnables. +5. **Pass the role**: when the app's `data.roles` gives a data table a role, every `wmill.datatable` call on it passes that role (`wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python). The role only reaches what it was granted, so a query outside it fails with `permission denied`. + +`wmill datatable list` lists the workspace's data tables. Create or change tables with a migration in `sql_to_apply/` (see "SQL Migrations" above), then add them to `data.tables` in `raw_app.yaml`. ### Querying in TypeScript (Bun/Deno) diff --git a/system_prompts/auto-generated/skills/resources/SKILL.md b/system_prompts/auto-generated/skills/resources/SKILL.md index 396b3b9d8e..bd690b1dec 100644 --- a/system_prompts/auto-generated/skills/resources/SKILL.md +++ b/system_prompts/auto-generated/skills/resources/SKILL.md @@ -52,6 +52,14 @@ Reference variables in resource values: - `$var:u/username/name` - User variable - `$var:f/folder/name` - Folder variable +## Secrets + +Never put a secret (password, API key, token) inline in a resource value. Store it in a secret variable and reference that variable as `$var:`. + +- `$var:` is a reference, not a value: it resolves to the variable's value at run time. Never invent a value for a variable. +- A resource that references a variable needs the variable to exist first, so create or deploy the variable before the resource. +- A secret's plaintext never goes in a file of the repo: a `.variable.yaml` holding it would be committed. Ask the user to create the secret on the workspace instead, e.g. `wmill variable add '' ` (a secret by default), then reference it by path. + ## Resource References Reference other resources: diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index f0f8b81a96..5c041585d4 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -85,7 +85,6 @@ An input typed as a resource (`format: resource-` in the schema) takes the To open the flow visually in the dev page (graph + live reload), use the `preview` skill. Always **offer** it as a one-sentence next step (e.g. "Want me to open the visual preview?") rather than opening it automatically — opening the dev page has side effects (browser window, possibly a `launch.json` entry under MCP-preview branches) the user should consent to. If the user already asked to see/preview/visualize the flow in their original request, skip the offer and just invoke the skill. - # Windmill Flow Building Guide ## OpenFlow Schema @@ -258,8 +257,8 @@ names, so neither is name-checked at all — leave those summaries as they are. - Always set `summary`. It must be unique among that agent's tools, and must not be one of the reserved ids (`do`, `bg`, `ctx`, `state`, `if`, `else`, `for`, `delete`, `while`, `new`, `in`, `failure`, `preprocessor`, `as`, `Input`, `Result`, `Trigger`) -- A tool name outside that character set is rejected: flow write tools refuse it, and a flow that - reaches the worker with one fails every run with `Invalid tool name` +- A tool name outside that character set fails any run that offers the tool to the agent, with `Invalid tool name`. + `wmill lint ` reports it before anything runs. - Tool `id` follows the same rules as any module ID — unique across the flow, underscores not spaces - `description` is optional free text telling the agent when and how to call the tool. Set it whenever the name alone does not make that obvious; it overrides the description derived from the @@ -282,9 +281,11 @@ names, so neither is name-checked at all — leave those summaries as they are. ## Loop Structure Rules +- A `forloopflow` runs its `modules` once per element of `iterator`, a javascript expression returning an array (e.g. `results.get_items`); `parallel: true` runs the iterations concurrently, and `skip_failures: true` carries on past a failed iteration - For `whileloopflow`, break the loop with a module-level `stop_after_if`: on the loop module itself, or on an inner step (required when that step carries state via its own `results` — see below) - `stop_after_if` is always a sibling of `id` and `value` on a flow module — never a direct key of the loop's `value` object - `stop_after_all_iters_if` is for checks after the whole loop finishes, not the normal per-iteration break condition +- `stop_after_if` is evaluated after each iteration: on the loop module, `result` is that iteration's result (what its last step returned); on an inner step, it is that step's result - `flow_input.iter.value` in a `whileloopflow` is just the iteration index (same number as `flow_input.iter.index`) — it never carries state, so `flow_input.iter.value.` is always undefined and a loop whose stop condition depends on it never terminates - To carry state across iterations, a step reads its own previous-iteration result via `results.` with a first-iteration fallback (e.g. `results.b ?? flow_input.start`) — but then the loop's `stop_after_if` MUST sit on that inner step, not on the loop module: a body that is exactly one plain step with the stop condition on the loop module runs on a fast path where `results.` is null on every iteration and the loop never terminates (bodies with 2+ steps, or whose single step has its own `stop_after_if`, retry or similar, resolve `results` across iterations regardless of stop placement) - For state that is just a counter, derive it from the index instead (e.g. `flow_input.iter.index + 1`) — that works in every configuration, including with `stop_after_if` on the loop module @@ -422,6 +423,7 @@ Incorrect shape (identity has no resume URLs — not a real approval): ## Branch Result Scope Rules +- A `branchone` runs the first of its `branches` whose `expr` is true, in order, and its `default` modules when none is; a `branchall` runs every branch (concurrently with `parallel: true`) - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch - Outside a `branchone`, do NOT reference ids of steps that only exist inside its branches or default branch. Use `results.` instead - Outside a `branchall`, do NOT reference ids of steps inside its branches. Use `results.` instead @@ -481,6 +483,41 @@ JavaScript transform (dynamic expression): - For flow inputs: Use type `"object"` with format `"resource-{type}"` (e.g., `"resource-postgresql"`) - For step inputs: Use static value `"$res:path/to/resource"` +## Reusing Existing Scripts and Flows + +Unless the user asked for new code, look for a workspace script or flow that already does a step's job before writing it, and reuse it by path instead of copying its logic into a rawscript: + +- a workspace script: `type: script` with `path` (e.g. `f/folder/send_email`) +- a workspace flow, run as a subflow: `type: flow` with `path` +- a Hub script: `type: script` with a `hub///` path + +The step's `input_transforms` must cover the reused item's inputs, so read its input schema first. +Find candidates in the local tree (a `.script.yaml` sits next to each script and holds its input schema, a `flow.yaml` in each flow folder) and on the workspace with `wmill script list` / `wmill flow list`; `wmill script get ` and `wmill flow get ` show an item's details. + +## Organizing Flows: Groups and Notes + +Groups and notes shape how a flow reads in the editor; neither changes what it does. + +**Segment every non-trivial flow into groups without waiting to be asked.** Whenever a flow has more than a couple of steps, or consecutive steps form a stage ("fetch", "transform", "notify"), put them in a group, and aim for every meaningful step to belong to one. Use notes sparingly, for flow-wide information that belongs to no span of steps: the flow's purpose, key assumptions, warnings, TODOs. One note is usually enough; never label a run of steps with a note, which is what a group is for. + +`value.groups` lists the groups, each spanning the steps from `start_id` to `end_id`: + +- `start_id`, `end_id` (required): ids of the group's first and last step; the same id for both makes a one-step group +- `summary`: the group's title +- `note`: markdown shown under the title +- `color`: one of `yellow`, `blue`, `green`, `purple`, `pink`, `orange`, `red`, `cyan`, `lime`, `gray`, never a hex code or CSS color; leave it out and the editor picks one +- `autocollapse`: `true` shows the group collapsed by default + +The editor refuses to draw a flow whose groups break any of these rules: + +- `start_id` and `end_id` are steps of the same list: both top-level, or both in the same loop body or branch. A group can hold a loop or branch step whole, but cannot start outside one and end inside it +- `start_id` does not come after `end_id` in that list +- groups nest (one entirely inside another) but never partly overlap, and no two groups share both `start_id` and `end_id` +- groups hold ordinary steps only: never `preprocessor`, `failure`, `Input`, `Result`, `Trigger`, or an AI agent's tools + +`value.notes` lists sticky notes, each with a unique `id`, markdown `text`, a `color` from the same list, and `type: free`. The `group` note type is deprecated; use `value.groups` instead. +Give each note a `position` (`{ x, y }`) and a `size` (`{ width, height }`): the editor draws a note without them at the origin and cannot resize it. `x: -400` with `width: 275` places it beside the graph. + ## Final Structural Self-Check Before finalizing a flow, verify: @@ -490,6 +527,8 @@ Before finalizing a flow, verify: - any approval step has module-level `suspend` - no downstream step references inner branch step ids from outside the branch - every AI agent flowmodule tool has a unique `summary` made only of letters, numbers and underscores +- every group starts and ends on steps of the same list, start before end, nesting without partial overlap +- `wmill lint ` reports no error ## S3 Object Operations @@ -544,7 +583,6 @@ Reference a specific resource using `$res:` prefix: } ``` - ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window`: it is ignored when memory is off, and an older `auto` or\n`manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} diff --git a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md new file mode 100644 index 0000000000..064dc600ac --- /dev/null +++ b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md @@ -0,0 +1,125 @@ +--- +name: write-pipeline +description: MUST use when creating or modifying a data pipeline, a set of scripts marked `pipeline` and wired together by `on` / `materialize` annotations. +--- + +# Data pipeline authoring + +A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. + +## Default to DuckDB + DuckLake + +A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
    ` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: + +- `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). +- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. + +Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. + +## Storage prerequisites + +A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. +`wmill ducklake list` lists them. +Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if there is none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: + +- a workspace **admin** sets it up in Workspace settings → Object Storage (add an S3/Azure/GCS storage), then adds a DuckLake catalog on top of it; +- anyone **without admin rights** should ask a workspace admin to configure object storage + a DuckLake catalog. + +Never hand back a DuckLake pipeline that cannot run without flagging the missing storage and pointing to who sets it up. + +## What makes a script a pipeline node + +A script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** — `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: + +- `// on ` — declares an execution-DAG **input** (what triggers/feeds this node). `` is either: + - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `$res:f/folder/my_resource`, `volume://name/path`, or an S3 object (see the S3 storage-form rule below). + - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `amqp`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. **`data_upload` is special**: there is no trigger row — the node instead declares an **`S3Object` input parameter** fed by the auto-generated upload picker; it never hard-codes a key. Any language can be the `data_upload` node: + - Python (has `import wmill`): `def main(file: wmill.S3Object):` then `wmill.load_s3_file(file)`; TS (has `import * as wmill from "windmill-client"`): `export async function main(file: wmill.S3Object)`. Qualify the type as `wmill.S3Object` (or add `from wmill import S3Object` / `import { S3Object } from "windmill-client"`) — a bare `S3Object` is undefined. + - **DuckDB** takes the s3object arg via a `-- $ (s3object)` declaration and reads it directly, so a single DuckDB node can ingest **and** materialize: + ``` + -- pipeline + -- on data_upload + -- materialize ducklake://main/raw_uploads + -- $file (s3object) + SELECT * FROM read_csv($file) + ``` +- **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...` (managed DuckLake targets only — deploy rejects it beside a `dbt://` one), `// measure = [where ]` and `// dimension = ` (see "Declared metrics" below). + +## S3 object wiring (storage form matters) + +An `s3://` URI's first slashes select the **storage**, not part of the key — get this wrong and the producer/consumer edge silently won't connect: + +- `s3:///` (**triple** slash, empty first segment) = the **default** workspace storage. A downstream node reading or triggering on that object uses `s3:///` — e.g. DuckDB `-- on s3:///orders/2024.parquet` and `read_parquet('s3:///orders/2024.parquet')`. +- `s3:///` (**double** slash, non-empty first segment) = a **named secondary** storage called `` — so `s3://ingest/x` means storage `ingest`, key `x`, NOT key `ingest/x`. Only use this when the object genuinely lives in a configured secondary storage; never invent a bucket/storage name for a default-storage object (it breaks the edge). + +To make the producer side visible to lineage, a Python/TS node MUST pass the **`S3Object` form**, not a bare key string: Python `wmill.write_s3_file(wmill.S3Object(s3=""), data)` (or the import-free dict `{"s3": ""}`), TS `wmill.writeS3File({ s3: "" }, data)`. That records the default-storage asset `/`, which a downstream `s3:///` reader connects to (same key both sides). A bare `write_s3_file("", ...)` records **no** asset and produces **no** edge. Add `storage=""` only for a named secondary storage. + +The key must be a **string literal** — the graph parser is static and cannot follow a variable, f-string, or computed path, so `write_s3_file(wmill.S3Object(s3=key_var), ...)` records no edge. Inline the literal (`s3="events/user_events.parquet"`) on both the writing and reading node. The same rule applies to every asset URI in an annotation or SDK call (`ducklake://`, `datatable://`, `s3://`): write them literally, not via a variable. + +## Materialize (the managed output) + +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
    `). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> +> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. + +A managed `// materialize ducklake:///
    ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. (The opposite holds for the `dbt://` target above: there the node writes its own DDL and the strategies below do not apply.) Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- `// materialize append` → INSERT-append rows (incremental); +- `// materialize key=` → merge/upsert on ``. + +`// materialize manual ` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table). The `{partition}` token, usable in any asset URI **and** in the body SQL, is replaced at run time by the current partition's **identity string**: + +- To filter the source to the active slice on a time grain, use the runtime-injected macro: `WHERE wm_partition() = {partition}`. `wm_partition(ts)` buckets a timestamp in exactly the identity format the runtime uses for daily/hourly/weekly/monthly, so it always matches; never hand-write a `strftime` format. +- Do NOT write `= TIMESTAMP {partition}`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at run time. +- For `dynamic` partitioning the identity is the caller-supplied key (not a timestamp, no macro), so filter on it directly: `WHERE = {partition}`. + +`materialize` is an output **declaration** on a node — not a command. There is no "materialize run". + +## Declared metrics (`measure` / `dimension`) + +On a node that materializes a DuckLake table, `// measure = [where ]` names the canonical way to aggregate that table (e.g. `// measure revenue = sum(amount) where not is_refund`), and `// dimension = ` names a way to slice it (e.g. `// dimension region = region`, `// dimension month = date_trunc('month', ordered_at)`). They execute nothing: they are catalogued at deploy so the editor and other agents reuse the definition instead of re-deriving it and silently disagreeing. + +- Keep the predicate in the `where` clause rather than folding it into the aggregate: it is rendered as ` FILTER (WHERE )`, which is what lets two measures with different predicates share one GROUP BY. +- DuckLake-only, and only meaningful next to `// materialize`. +- Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures — an obvious `count(*)` earns nothing. +- To use a metric another node declares, read that node and reuse its exact expression rather than guessing it. + +## How to build one + +1. Put every node in the **same folder**: `f//`. The folder is the pipeline. +2. Write each node as its own script. Default to `duckdb` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick `postgresql`, `bun`, or `python3` only when that section says the work calls for it. +3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Don't deploy nodes unless the user asks to. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +Locally: + +- create each node with `wmill script new f// `, then write its body; +- `wmill pipeline show --local` draws the graph from your working tree — check that every edge you meant to form is there; +- `wmill pipeline dev ` live-previews the pipeline, and `wmill pipeline run --local` runs the cascade from local files without deploying (`--dry-run` prints the plan first); +- a trigger such as `// on schedule` only declares the binding: the schedule or trigger itself is created separately (see the `schedules` and `triggers` skills). + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table): + +```sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3:///raw/orders/*.csv') +``` + +Node `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup): + +```sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +``` diff --git a/system_prompts/auto-generated/skills/write-script-ansible/SKILL.md b/system_prompts/auto-generated/skills/write-script-ansible/SKILL.md index 2c8519a301..e6683248db 100644 --- a/system_prompts/auto-generated/skills/write-script-ansible/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-ansible/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Ansible Windmill runs Ansible playbooks with `ansible-playbook`. A script is a single YAML diff --git a/system_prompts/auto-generated/skills/write-script-bash/SKILL.md b/system_prompts/auto-generated/skills/write-script-bash/SKILL.md index 6931a0ed58..42bbd7bc01 100644 --- a/system_prompts/auto-generated/skills/write-script-bash/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bash/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Bash ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md b/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md index 1127373c3f..dc3e19c0c9 100644 --- a/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bigquery/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # BigQuery Arguments use `@name` syntax. diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 1d52d0a8a1..6d82183b88 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # TypeScript (Bun) Bun runtime with full npm ecosystem and fastest execution. **Bun is the default and preferred TypeScript runtime** — choose it for any TypeScript script unless there is a major reason to use Deno for that specific use-case. @@ -102,7 +125,7 @@ import * as wmill from "windmill-client"; **Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method to use instead of guessing or falling back to `fetch`. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method to use instead of guessing or falling back to `fetch`. ## Preprocessor Scripts @@ -120,7 +143,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -168,7 +193,6 @@ const result: wmill.S3Object = await wmill.writeS3File( ); ``` - # TypeScript SDK (windmill-client) Import: import * as wmill from 'windmill-client' diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index 4943c11baa..a7d98c0acc 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # TypeScript (Bun Native) Native TypeScript execution. Native scripts are Bun scripts that run on the native worker — a lightweight V8 isolate that exposes `fetch` and the JavaScript standard library — and can be heavily parallelized. Every script MUST start with `//native` on its first line so Windmill routes it to the native worker; without it the exact same script runs on the regular Bun worker. You may import npm packages and other Windmill scripts (e.g. `./helper.ts`) — imports are resolved and bundled just like a regular Bun script — as long as everything (your code and its dependencies) relies only on `fetch` and the standard library. Libraries that need Node/Bun runtime APIs (filesystem, `node:*` modules, child processes, native addons) will not work on the native worker; use the regular `bun` language for those. @@ -99,7 +122,7 @@ export async function main(url: string) { `windmill-client` works on the native worker (its calls go over `fetch`), so use it as the **preferred way to talk to Windmill** — reading resources/variables/states, running scripts and flows, and the S3 helpers below (`loadS3File`, `loadS3FileStream`, `writeS3File`, `S3Object`). It handles auth, the workspace, and the base URL for you. Reserve raw `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API. ## Preprocessor Scripts @@ -118,7 +141,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -168,7 +193,6 @@ const result: wmill.S3Object = await wmill.writeS3File( ); ``` - # TypeScript SDK (windmill-client) Import: import * as wmill from 'windmill-client' diff --git a/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md b/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md index 95adb7c71e..d755483abb 100644 --- a/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-csharp/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # C# The script must contain a public static `Main` method inside a class: diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index fcc2d68847..eec2f25dfa 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # TypeScript (Deno) Deno runtime with npm support via `npm:` prefix and native Deno libraries. @@ -104,7 +127,7 @@ import * as wmill from "windmill-client"; **Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of guessing or falling back to `fetch`. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method instead of guessing or falling back to `fetch`. ## Preprocessor Scripts @@ -122,7 +145,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; @@ -170,7 +195,6 @@ const result: wmill.S3Object = await wmill.writeS3File( ); ``` - # TypeScript SDK (windmill-client) Import: import * as wmill from 'windmill-client' diff --git a/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md b/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md index e732183c22..b2b841d0eb 100644 --- a/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-duckdb/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # DuckDB Arguments are defined with comments and used with `$name` syntax: diff --git a/system_prompts/auto-generated/skills/write-script-go/SKILL.md b/system_prompts/auto-generated/skills/write-script-go/SKILL.md index b089418a3c..31a60a7548 100644 --- a/system_prompts/auto-generated/skills/write-script-go/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-go/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Go ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md b/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md index 377932b1f0..9924e72f7f 100644 --- a/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-graphql/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # GraphQL ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-java/SKILL.md b/system_prompts/auto-generated/skills/write-script-java/SKILL.md index fdccdba79f..a28d7763cb 100644 --- a/system_prompts/auto-generated/skills/write-script-java/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-java/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Java The script must contain a Main public class with a `public static main()` method: diff --git a/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md b/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md index f8536ce963..75f3827440 100644 --- a/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-mssql/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Microsoft SQL Server (MSSQL) Arguments use `@P1`, `@P2`, etc. diff --git a/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md b/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md index 4e09271274..c2f034c4f6 100644 --- a/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-mysql/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # MySQL Arguments use `?` placeholders. diff --git a/system_prompts/auto-generated/skills/write-script-php/SKILL.md b/system_prompts/auto-generated/skills/write-script-php/SKILL.md index efa7ded59a..1052944b8e 100644 --- a/system_prompts/auto-generated/skills/write-script-php/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-php/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # PHP ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md b/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md index 2f1e495277..8937c078f4 100644 --- a/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-postgresql/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # PostgreSQL Arguments are obtained directly in the statement with `$1::{type}`, `$2::{type}`, etc. diff --git a/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md b/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md index 9b729ce87a..c3e2bd7500 100644 --- a/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-powershell/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # PowerShell ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index 42916f7af5..b030ff4316 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Python ## Structure @@ -131,7 +154,7 @@ For preprocessor scripts, the function should be named `preprocessor` and receiv from typing import TypedDict, Literal, Any class Event(TypedDict): - kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "amqp", "gcp", "azure"] body: Any headers: dict[str, str] query: dict[str, str] @@ -182,7 +205,6 @@ result: S3Object = wmill.write_s3_file( ) ``` - # Python SDK (wmill) Import: import wmill @@ -979,4 +1001,3 @@ async def parallel(items, fn, *, concurrency: Optional[int] = None) # partition: Partition number (from event['partition']) # offset: Message offset to commit (from event['offset']) def commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None - diff --git a/system_prompts/auto-generated/skills/write-script-rlang/SKILL.md b/system_prompts/auto-generated/skills/write-script-rlang/SKILL.md index b90ccb3bf8..626b602749 100644 --- a/system_prompts/auto-generated/skills/write-script-rlang/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-rlang/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # R ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-rust/SKILL.md b/system_prompts/auto-generated/skills/write-script-rust/SKILL.md index b28951e7bc..0f4a95bbe2 100644 --- a/system_prompts/auto-generated/skills/write-script-rust/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-rust/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Rust ## Structure diff --git a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md index bf428e2d8f..99af11e419 100644 --- a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md @@ -48,6 +48,29 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. +# Windmill Script Writing Guide + +## General Principles + +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters +- Libraries are installed automatically - do not show installation instructions +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform + +## Return Values + +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces +- Return values become available to subsequent flow steps via `results.step_id` + +## Preprocessor Scripts + +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). + +The returned object determines the parameter values passed to the flow. +e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. + # Snowflake Arguments use `?` placeholders. diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index 8e5ab1f06c..13d305c74e 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -48,14 +48,13 @@ For a **visual** open-the-script-in-the-dev-page preview (rather than `script pr Use `wmill resource-type list --schema` to discover available resource types. -Workflow-as-Code files use the normal script CLI workflow. There are no separate WAC deploy commands. - # Windmill Workflow-as-Code Writing Guide ## Scope Use this guide when writing or modifying Windmill Workflow-as-Code (WAC) scripts. WAC is authored as a Windmill script and deployed with the normal script workflow. It is not an OpenFlow YAML flow. +Workflow-as-Code files use the normal script CLI workflow. There are no separate WAC deploy commands. Supported WAC authoring targets: - Bun TypeScript scripts that import from `windmill-client` @@ -240,7 +239,6 @@ TypeScript: avoid broad `try/catch` around WAC SDK calls. The SDK uses an intern A caught failure reads the same whether it came from a task or from a `step()`, and the same in the round that ran the failing body as in every round replaying it. It carries `step_key`, `child_job_id` (absent for a `step()`, which runs in the workflow job and has no child job), a `message` that is the failure's own message, and `result` = `{"error": {"name", "message", "stack"?, "extra"?}}`. `name`, `message` and `stack` are the fields that read the same whichever side failed; `name` and `message` are always there, `stack` only when the failure had a traceback to give. `extra` carries the failure's own custom fields (an exception's attributes, an error's properties) and is best-effort: it is absent when there were none, and a task can report entries a step does not, so read it defensively and don't branch on its absence. `extra` is dropped when it is too large to keep in the checkpoint, and `extra_omitted: true` says so — absent `extra` with no `extra_omitted` means the failure simply had no custom fields. Branch on those, not on the original exception type: the workflow body re-runs from the top every round and a replay rebuilds the failure from the checkpoint, so nothing outside that record survives. Python raises `TaskError`; TypeScript throws an `Error` named `TaskError` carrying the same fields. Nothing is chained onto `__cause__` / `cause` — the traceback is in `result.error.stack`, and is also printed to the job log when the step fails. - ## TypeScript Workflow-as-Code API (windmill-client) Import: `import { workflow, task, taskScript, taskFlow, step, sleep, waitForApproval, getApprovalUrls, getResumeUrls, parallel } from "windmill-client"` diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index 54441dd96b..ee25a5ce3d 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -170,8 +170,13 @@ names, so neither is name-checked at all — leave those summaries as they are. - Always set `summary`. It must be unique among that agent's tools, and must not be one of the reserved ids (`do`, `bg`, `ctx`, `state`, `if`, `else`, `for`, `delete`, `while`, `new`, `in`, `failure`, `preprocessor`, `as`, `Input`, `Result`, `Trigger`) -- A tool name outside that character set is rejected: flow write tools refuse it, and a flow that - reaches the worker with one fails every run with `Invalid tool name` +- A tool name outside that character set fails any run that offers the tool to the agent, with `Invalid tool name`. + + The flow write tools refuse such a name. + + + `wmill lint ` reports it before anything runs. + - Tool `id` follows the same rules as any module ID — unique across the flow, underscores not spaces - `description` is optional free text telling the agent when and how to call the tool. Set it whenever the name alone does not make that obvious; it overrides the description derived from the @@ -194,9 +199,11 @@ names, so neither is name-checked at all — leave those summaries as they are. ## Loop Structure Rules +- A `forloopflow` runs its `modules` once per element of `iterator`, a javascript expression returning an array (e.g. `results.get_items`); `parallel: true` runs the iterations concurrently, and `skip_failures: true` carries on past a failed iteration - For `whileloopflow`, break the loop with a module-level `stop_after_if`: on the loop module itself, or on an inner step (required when that step carries state via its own `results` — see below) - `stop_after_if` is always a sibling of `id` and `value` on a flow module — never a direct key of the loop's `value` object - `stop_after_all_iters_if` is for checks after the whole loop finishes, not the normal per-iteration break condition +- `stop_after_if` is evaluated after each iteration: on the loop module, `result` is that iteration's result (what its last step returned); on an inner step, it is that step's result - `flow_input.iter.value` in a `whileloopflow` is just the iteration index (same number as `flow_input.iter.index`) — it never carries state, so `flow_input.iter.value.` is always undefined and a loop whose stop condition depends on it never terminates - To carry state across iterations, a step reads its own previous-iteration result via `results.` with a first-iteration fallback (e.g. `results.b ?? flow_input.start`) — but then the loop's `stop_after_if` MUST sit on that inner step, not on the loop module: a body that is exactly one plain step with the stop condition on the loop module runs on a fast path where `results.` is null on every iteration and the loop never terminates (bodies with 2+ steps, or whose single step has its own `stop_after_if`, retry or similar, resolve `results` across iterations regardless of stop placement) - For state that is just a counter, derive it from the index instead (e.g. `flow_input.iter.index + 1`) — that works in every configuration, including with `stop_after_if` on the loop module @@ -334,6 +341,7 @@ Incorrect shape (identity has no resume URLs — not a real approval): ## Branch Result Scope Rules +- A `branchone` runs the first of its `branches` whose `expr` is true, in order, and its `default` modules when none is; a `branchall` runs every branch (concurrently with `parallel: true`) - Inside a branch, you may reference earlier outer steps and earlier steps in the same branch - Outside a `branchone`, do NOT reference ids of steps that only exist inside its branches or default branch. Use `results.` instead - Outside a `branchall`, do NOT reference ids of steps inside its branches. Use `results.` instead @@ -393,6 +401,48 @@ JavaScript transform (dynamic expression): - For flow inputs: Use type `"object"` with format `"resource-{type}"` (e.g., `"resource-postgresql"`) - For step inputs: Use static value `"$res:path/to/resource"` +## Reusing Existing Scripts and Flows + +Unless the user asked for new code, look for a workspace script or flow that already does a step's job before writing it, and reuse it by path instead of copying its logic into a rawscript: + +- a workspace script: `type: script` with `path` (e.g. `f/folder/send_email`) +- a workspace flow, run as a subflow: `type: flow` with `path` +- a Hub script: `type: script` with a `hub///` path + +The step's `input_transforms` must cover the reused item's inputs, so read its input schema first. + +Find candidates in the local tree (a `.script.yaml` sits next to each script and holds its input schema, a `flow.yaml` in each flow folder) and on the workspace with `wmill script list` / `wmill flow list`; `wmill script get ` and `wmill flow get ` show an item's details. + + +## Organizing Flows: Groups and Notes + +Groups and notes shape how a flow reads in the editor; neither changes what it does. + +**Segment every non-trivial flow into groups without waiting to be asked.** Whenever a flow has more than a couple of steps, or consecutive steps form a stage ("fetch", "transform", "notify"), put them in a group, and aim for every meaningful step to belong to one. Use notes sparingly, for flow-wide information that belongs to no span of steps: the flow's purpose, key assumptions, warnings, TODOs. One note is usually enough; never label a run of steps with a note, which is what a group is for. + +`value.groups` lists the groups, each spanning the steps from `start_id` to `end_id`: + +- `start_id`, `end_id` (required): ids of the group's first and last step; the same id for both makes a one-step group +- `summary`: the group's title +- `note`: markdown shown under the title +- `color`: one of `yellow`, `blue`, `green`, `purple`, `pink`, `orange`, `red`, `cyan`, `lime`, `gray`, never a hex code or CSS color; leave it out and the editor picks one +- `autocollapse`: `true` shows the group collapsed by default + +The editor refuses to draw a flow whose groups break any of these rules: + +- `start_id` and `end_id` are steps of the same list: both top-level, or both in the same loop body or branch. A group can hold a loop or branch step whole, but cannot start outside one and end inside it +- `start_id` does not come after `end_id` in that list +- groups nest (one entirely inside another) but never partly overlap, and no two groups share both `start_id` and `end_id` +- groups hold ordinary steps only: never `preprocessor`, `failure`, `Input`, `Result`, `Trigger`, or an AI agent's tools + +`value.notes` lists sticky notes, each with a unique `id`, markdown `text`, a `color` from the same list, and `type: free`. The `group` note type is deprecated; use `value.groups` instead. + +Give each note a `position` (`{ x, y }`) and a `size` (`{ width, height }`): the editor draws a note without them at the origin and cannot resize it. `x: -400` with `width: 275` places it beside the graph. + + +Leave a note's `position` and `size` out and they are filled in for you. + + ## Final Structural Self-Check Before finalizing a flow, verify: @@ -402,6 +452,10 @@ Before finalizing a flow, verify: - any approval step has module-level `suspend` - no downstream step references inner branch step ids from outside the branch - every AI agent flowmodule tool has a unique `summary` made only of letters, numbers and underscores +- every group starts and ends on steps of the same list, start before end, nesting without partial overlap + +- `wmill lint ` reports no error + ## S3 Object Operations diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index 3f2d985de4..01e9797e32 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -13,7 +13,11 @@ Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lak ## Storage prerequisites -A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check with the `list_ducklakes` tool before you build (it returns the configured DuckLake catalogs, or none). Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if `list_ducklakes` returns none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: +A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. + +`wmill ducklake list` lists them. + +Drafting the annotated scripts does not require storage, but the pipeline can't ingest, materialize, or read its assets until it exists. So if there is none (or the user hits "storage not configured" errors), say so and give the right next step **by role**: - a workspace **admin** sets it up in Workspace settings → Object Storage (add an S3/Azure/GCS storage), then adds a DuckLake catalog on top of it; - anyone **without admin rights** should ask a workspace admin to configure object storage + a DuckLake catalog. @@ -37,7 +41,7 @@ A script joins the pipeline when its source begins with the `pipeline` annotatio SELECT * FROM read_csv($file) ``` - **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. -- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...` (managed DuckLake targets only — deploy rejects it beside a `dbt://` one). +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...` (managed DuckLake targets only — deploy rejects it beside a `dbt://` one), `// measure = [where ]` and `// dimension = ` (see "Declared metrics" below). ## S3 object wiring (storage form matters) @@ -64,19 +68,39 @@ A managed `// materialize ducklake:///
    ` tells the runtime to write `// materialize manual ` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. -`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time. +`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table). The `{partition}` token, usable in any asset URI **and** in the body SQL, is replaced at run time by the current partition's **identity string**: + +- To filter the source to the active slice on a time grain, use the runtime-injected macro: `WHERE wm_partition() = {partition}`. `wm_partition(ts)` buckets a timestamp in exactly the identity format the runtime uses for daily/hourly/weekly/monthly, so it always matches; never hand-write a `strftime` format. +- Do NOT write `= TIMESTAMP {partition}`: the identity string is not a valid timestamp literal for hourly/weekly/monthly and errors at run time. +- For `dynamic` partitioning the identity is the caller-supplied key (not a timestamp, no macro), so filter on it directly: `WHERE = {partition}`. `materialize` is an output **declaration** on a node — not a command. There is no "materialize run". -## How to build one in chat +## Declared metrics (`measure` / `dimension`) + +On a node that materializes a DuckLake table, `// measure = [where ]` names the canonical way to aggregate that table (e.g. `// measure revenue = sum(amount) where not is_refund`), and `// dimension = ` names a way to slice it (e.g. `// dimension region = region`, `// dimension month = date_trunc('month', ordered_at)`). They execute nothing: they are catalogued at deploy so the editor and other agents reuse the definition instead of re-deriving it and silently disagreeing. + +- Keep the predicate in the `where` clause rather than folding it into the aggregate: it is rendered as ` FILTER (WHERE )`, which is what lets two measures with different predicates share one GROUP BY. +- DuckLake-only, and only meaningful next to `// materialize`. +- Declare one when a number carries a judgement call someone else would get wrong (refunds excluded, test rows dropped, which column is the amount); do NOT blanket every table with measures — an obvious `count(*)` earns nothing. +- To use a metric another node declares, read that node and reuse its exact expression rather than guessing it. + +## How to build one 1. Put every node in the **same folder**: `f//`. The folder is the pipeline. -2. Author each node as a **script draft** with `write_script` (or `edit_script`). Default to `duckdb` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick `postgresql`, `bun`, or `python3` only when that section says the work calls for it. +2. Write each node as its own script. Default to `duckdb` materializing into DuckLake (see "Default to DuckDB + DuckLake" above); pick `postgresql`, `bun`, or `python3` only when that section says the work calls for it. 3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. 4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. -5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. +5. Don't deploy nodes unless the user asks to. A pipeline only "runs" once its scripts are deployed and their triggers exist. + -When the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. +Locally: + +- create each node with `wmill script new f// `, then write its body; +- `wmill pipeline show --local` draws the graph from your working tree — check that every edge you meant to form is there; +- `wmill pipeline dev ` live-previews the pipeline, and `wmill pipeline run --local` runs the cascade from local files without deploying (`--dry-run` prints the plan first); +- a trigger such as `// on schedule` only declares the binding: the schedule or trigger itself is created separately (see the `schedules` and `triggers` skills). + ## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) diff --git a/system_prompts/base/raw-app-cli.md b/system_prompts/base/raw-app-cli.md index 030b958379..0ff11a2b74 100644 --- a/system_prompts/base/raw-app-cli.md +++ b/system_prompts/base/raw-app-cli.md @@ -163,7 +163,7 @@ The `data` block in `raw_app.yaml` controls which tables the app can query. ```yaml data: datatable: main # Default datatable - schema: app_schema # Default schema (optional) + schema: app_schema # Schema the app's tables go in (optional); still write them as app_schema.
    tables: - main/users # Table in public schema - main/app_schema:items # Table in specific schema @@ -208,6 +208,8 @@ data: - main/users ``` +A migration runs with no default schema, so a table outside `public` is created with its schema, `CREATE TABLE IF NOT EXISTS app_schema.items (...)`, listed as `main/app_schema:items`, and queried as `app_schema.items`. + ### Migration best practices - **Use idempotent SQL**: `CREATE TABLE IF NOT EXISTS`, etc. @@ -217,8 +219,9 @@ data: ## CLI Commands -Two commands you run yourself, not the user: +Commands you run yourself, not the user: - `wmill app new` — run it with flags, per the "Creating a Raw App" section above. +- `wmill app lint ` — checks the app's structure and that it builds. Run it after editing, before offering a preview or a deploy; a bundle that compiles still says nothing about behavior, so a preview is what checks that. - `wmill generate-metadata` — (re)generates local lock files and refreshes `wmill-lock.yaml` content hashes; writes local files only (not a deploy). After adding or editing a runnable, offer it and run it on agreement — or automatically if the project's `AGENTS.md` opts into that (see "After creating a runnable" above). For the rest, tell the user which command fits their intent and let them run it — these deploy to the workspace, overwrite local files, or launch a long-running server, so the user should consent each time: diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index f7eaef944d..56c6e900d4 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -48,6 +48,8 @@ Import the generated bindings and call the runnable like a function. `./wmill` i | `getJob(jobId)` | a `Job` (`{ type, success, result, duration_ms, ... }`) | polling status without blocking | | `streamJob(jobId, onUpdate?)` | the final result, calling `onUpdate` per chunk | showing output as it is produced | +A runnable is always called with **one object** whose keys are its `main` parameters — `main(user_id: string, limit: number)` is called as `backend.get_users({ user_id, limit })`, never with positional arguments. A runnable without parameters is called with no argument. Resource and variable ids handed to the `wmill` client are paths (`u//` or `f//`). + Run and wait — the common case: ```tsx @@ -121,9 +123,12 @@ Each runnable has a unique key (used to call it from the frontend) and one of fo ### Inline runnables -Inline runnables carry their own source code. For file-based raw apps, the runnable language is determined by the backend file extension. The script must expose a `main` function as its entrypoint. +Inline runnables carry their own source code, and must expose a `main` function as their entrypoint. + +On disk, a runnable's language is determined by its backend file extension. + -**TypeScript example** (`backend/get_user.ts`): +**TypeScript example** (runnable `get_user`): ```typescript import * as wmill from 'windmill-client'; @@ -135,7 +140,7 @@ export async function main(user_id: string) { } ``` -**Python example** (`backend/get_user.py`): +**Python example** (runnable `get_user`): ```python import wmill @@ -152,12 +157,14 @@ An inline runnable runs as an ordinary Windmill job. `import * as wmill from 'wi **Don't read `WM_TOKEN` or `BASE_INTERNAL_URL` and build an API URL to `fetch`.** The client's own `setClient` already reads exactly those, and it also sets the credentials mode a raw app needs (`WM_RAW_APP` suppresses credentials, because a sandboxed bundle calls the API from an opaque origin that can never pair with `Access-Control-Allow-Origin: *`). Rebuilding that by hand drops the parts you can't see. Use `wmill.*` for everything Windmill, and `fetch` only for third-party APIs. -Prefer the `wmill` functions that appear in the SDK reference; for an endpoint none of them covers, the generated service classes (`JobService`, `ScriptService`, ...) are importable from `windmill-client`. What is not available is a name you guessed at: `getBaseUrl` and `getWorkspaceToken` are inventions, not API. +Use only `wmill` functions the SDK actually exports; for an endpoint none of them covers, the generated service classes (`JobService`, `ScriptService`, ...) are importable from `windmill-client`. What is not available is a name you guessed at: `getBaseUrl` and `getWorkspaceToken` are inventions, not API. ### Path runnables (script / flow / hubscript) When `type` is `script`, `flow`, or `hubscript`, the runnable just stores a `path` to an existing workspace or hub item — no inline code. The referenced item's input/output schema becomes the runnable's surface. +Before writing an inline runnable, look for a workspace script or flow that already does the job, or a Hub script (a prebuilt integration at `hub///`) for a third-party service, and reference it instead of copying its logic. + ### Draft code vs deployed code This decides whether an app works before anything is deployed: @@ -177,15 +184,36 @@ Prefer a **path runnable of type `flow`** over an inline runnable that calls `wm `staticInputs` is an optional `Record` for arguments not overridable from the frontend. Useful with path runnables to pre-fill some args while leaving the rest to the frontend caller. +## Who can open a deployed app + +A draft app is reachable by nobody; deploying is what exposes it, and its backend runnables with it. Otherwise only users with access to the app can open it, unless the app is opened to: + +- **anonymous** users: anyone with the URL, without logging in; +- **guests**: anyone the instance's identity provider authenticates, whether or not they belong to the workspace. + +Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. + +`public: true` in `raw_app.yaml` deploys the app for anonymous users, and `guests: true` for guests. Remove the line and the next push closes the app again. + + +When a deploy widens who may open the app, tell the user in plain words. + + ## Data Tables -Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. +Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. ### Critical rules -1. **Whitelisted tables only**: a runnable can only query tables listed in the app's `data.tables` config. Tables not in this list are not accessible. -2. **Add tables before using**: queries against unlisted tables fail at runtime. When you introduce a new table, register it in `data.tables` first. -3. **Use the configured datatable/schema**: the app's `data` config sets the default datatable and schema; reference them consistently across runnables. +1. **Check what exists first**: look up the workspace's data tables and their tables before designing storage, and reuse a suitable table rather than creating another. Never assume a `main` data table exists. +2. **Whitelisted tables only**: a runnable can only query tables listed in the app's `data.tables` config. Queries against unlisted tables fail at runtime, so register a new table there before using it. +3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. +4. **Qualify table names**: an unqualified name means the `public` schema, so write every other table as `schema.table`, in table creation and queries alike. The app's `data` config sets the default datatable and schema its tables go in; use them consistently across runnables. +5. **Pass the role**: when the app's `data.roles` gives a data table a role, every `wmill.datatable` call on it passes that role (`wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python). The role only reaches what it was granted, so a query outside it fails with `permission denied`. + + +`wmill datatable list` lists the workspace's data tables. Create or change tables with a migration in `sql_to_apply/` (see "SQL Migrations" above), then add them to `data.tables` in `raw_app.yaml`. + ### Querying in TypeScript (Bun/Deno) diff --git a/system_prompts/base/resources.md b/system_prompts/base/resources.md index 5252683a81..2929051a03 100644 --- a/system_prompts/base/resources.md +++ b/system_prompts/base/resources.md @@ -2,11 +2,13 @@ Resources store credentials and configuration for external services. + ## File Format Resource files use the pattern: `{path}.resource.json` Example: `f/databases/postgres_prod.resource.json` + ## Resource Structure @@ -47,6 +49,16 @@ Reference variables in resource values: - `$var:u/username/name` - User variable - `$var:f/folder/name` - Folder variable +## Secrets + +Never put a secret (password, API key, token) inline in a resource value. Store it in a secret variable and reference that variable as `$var:`. + +- `$var:` is a reference, not a value: it resolves to the variable's value at run time. Never invent a value for a variable. +- A resource that references a variable needs the variable to exist first, so create or deploy the variable before the resource. + +- A secret's plaintext never goes in a file of the repo: a `.variable.yaml` holding it would be committed. Ask the user to create the secret on the workspace instead, e.g. `wmill variable add '' ` (a secret by default), then reference it by path. + + ## Resource References Reference other resources: @@ -260,6 +272,7 @@ def main(db: postgresql): pass ``` + ## CLI Commands ```bash @@ -277,3 +290,4 @@ wmill resource-type get postgresql # deploy via `git push` or `wmill sync push` (see the Deploying section in AGENTS.wmill.md). wmill sync push ``` + diff --git a/system_prompts/base/script-base.md b/system_prompts/base/script-base.md index 71ffada86e..2641d7cd0a 100644 --- a/system_prompts/base/script-base.md +++ b/system_prompts/base/script-base.md @@ -2,26 +2,21 @@ ## General Principles -- Scripts must export a main function (do not call it) +- A script's inputs are its parameters. Credentials and configuration come in as resource-typed parameters, never hard-coded or read from the environment; the language section below shows how that language declares parameters - Libraries are installed automatically - do not show installation instructions -- Credentials and configuration are stored in resources and passed as parameters -- The windmill client (`wmill`) provides APIs for interacting with the platform - -## Function Naming - -- Main function: `main` (or `preprocessor` for preprocessor scripts) -- Must be async for TypeScript variants +- In a language with an entrypoint function (TypeScript, Python, Go, Rust, PHP, R, …), name it `main` (`Main` in C#) and do not call it; in TypeScript it must be async. SQL, GraphQL, Bash, PowerShell and Ansible scripts have no `main`: their language section shows how they take arguments +- Where the language has a Windmill client (`wmill`), use it to interact with the platform ## Return Values -- Scripts can return any JSON-serializable value +- A script can return any JSON-serializable value; a SQL script returns the rows its query produces - Return values become available to subsequent flow steps via `results.step_id` ## Preprocessor Scripts -Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. +Preprocessor scripts process raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, AMQP, Postgres, GCP Pub/Sub, Azure, or email) before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. + +A preprocessor is written in TypeScript or Python: its function is named `preprocessor` instead of `main`, and it receives a single parameter called `event` (the language section gives its type). The returned object determines the parameter values passed to the flow. e.g., `{ b: 1, a: 2 }` calls the flow with `a = 2` and `b = 1`, assuming the flow has two inputs called `a` and `b`. - -The preprocessor receives a single parameter called `event`. diff --git a/system_prompts/base/script-cli.md b/system_prompts/base/script-cli.md new file mode 100644 index 0000000000..136a23172c --- /dev/null +++ b/system_prompts/base/script-cli.md @@ -0,0 +1,44 @@ +## CLI Commands + +Place scripts in a folder. + +After writing, tell the user which command fits what they want to do: + +- `wmill script preview ` — **default when iterating on a local script.** Runs the local file without deploying. +- `wmill script run ` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. +- `wmill generate-metadata` — regenerate the local `.script.yaml` (input schema) and `.lock` (resolved dependencies) for scripts you changed, and refresh their content hashes in `wmill-lock.yaml`. Local files only — **not** a deploy. See "Keep metadata in sync" below. +- Deploy local changes to the workspace — via `git push` or `wmill sync push` depending on how the repo is wired (see the **Deploying** section in `AGENTS.wmill.md`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". + +### Preview vs run — choose by intent, not habit + +If the user says "run the script", "try it", "test it", "does it work" while there are **local edits to the script file**, use `script preview`. Do NOT push the script to then `script run` it — pushing is a deploy, and deploying just to test overwrites the workspace version with untested changes. + +Only use `script run` when: +- The user explicitly says "run the deployed version" / "run what's on the server". +- There is no local script being edited (you're just invoking an existing script). + +Only use `sync push` when: +- The user explicitly asks to deploy, publish, push, or ship. +- The preview has already validated the change and the user wants it in the workspace. + +### Keep metadata in sync after editing + +`wmill-lock.yaml` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing `main`'s arguments** — invalidates that hash and leaves the `.lock`, the `.script.yaml` input schema, and the hash row out of date. Run `wmill generate-metadata` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by `.script.yaml`), and `wmill-lock.yaml` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. + +This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's `AGENTS.md` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated `.lock` / `.script.lock` files and tell the user which dependency versions changed (e.g. `requests 2.31.0 → 2.32.0`), so they can catch an unwanted bump before deploying — even under `Metadata: auto`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. + +With no path argument, `generate-metadata` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run `wmill generate-metadata --dry-run` — it lists each stale item with a reason (`content changed` or `depends on `) without changing anything — then narrow with a path argument (`wmill generate-metadata f/foo`) or `--strict-folder-boundaries`. + +If the on-disk `.lock` and `.script.yaml` are already correct and only `wmill-lock.yaml` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use `wmill generate-metadata rehash` — it re-records hashes from disk with no backend round-trip and no dependency changes. + +### After writing — offer to test, don't wait passively + +If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run `wmill script preview` with sample args?"). Do not present a multi-option menu. + +If the user already asked to test/run/try the script in their original request, skip the offer and just execute `wmill script preview -d ''` directly — pick plausible args from the script's declared parameters. The shape varies by language: `main(...)` for code languages, the SQL dialect's own placeholder syntax (`$1` for PostgreSQL, `?` for MySQL/Snowflake, `@P1` for MSSQL, `@name` for BigQuery, etc.), positional `$1`, `$2`, … for Bash, `param(...)` for PowerShell. + +`wmill script preview` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). `wmill generate-metadata` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's `AGENTS.md` opts in), per "Keep metadata in sync" above. Deploying to the workspace (`git push` or `wmill sync push` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. + +For a **visual** open-the-script-in-the-dev-page preview (rather than `script preview`'s run-and-print-result), use the `preview` skill. + +Use `wmill resource-type list --schema` to discover available resource types. diff --git a/system_prompts/base/workflow-as-code.md b/system_prompts/base/workflow-as-code.md index 9a1eea4700..0e529f6803 100644 --- a/system_prompts/base/workflow-as-code.md +++ b/system_prompts/base/workflow-as-code.md @@ -4,6 +4,9 @@ Use this guide when writing or modifying Windmill Workflow-as-Code (WAC) scripts. WAC is authored as a Windmill script and deployed with the normal script workflow. It is not an OpenFlow YAML flow. + +Workflow-as-Code files use the normal script CLI workflow. There are no separate WAC deploy commands. + Supported WAC authoring targets: - Bun TypeScript scripts that import from `windmill-client` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 2860b9d15c..918deef0fb 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -51,6 +51,7 @@ from utils import ( clean_params, escape_for_ts, read_markdown_file, + render_for, # Parsing utilities extract_balanced, extract_return_type, @@ -1619,88 +1620,165 @@ def extract_wac_py_sdk(py_content: str) -> str: # ============================================================================= -def generate_skill_content( - skill_name: str, - description: str, - intro: str, - content: str, - sdk_content: str = '' -) -> str: +def generate_skill_content(skill_name: str, description: str, body: str) -> str: """Generate a skill file with YAML frontmatter.""" - parts = [ - "---", - f"name: {skill_name}", - f"description: {description}", - "---", - "", - ] - if intro: - parts.extend([intro, ""]) - parts.append(content) - if sdk_content: - parts.extend(["", sdk_content]) - return '\n'.join(parts) + frontmatter = f"name: {skill_name}\ndescription: {description}\n" + # Agents find a skill by parsing this frontmatter, so a description YAML reads + # differently (a `: `, a leading quote or bracket) must fail here, not at load time. + try: + parsed = yaml.safe_load(frontmatter) + except yaml.YAMLError as e: + raise ValueError(f"Skill '{skill_name}' frontmatter is not valid YAML: {e}") from e + if parsed != {'name': skill_name, 'description': description}: + raise ValueError(f"Skill '{skill_name}' description does not read back as plain YAML text") + return f"---\n{frontmatter}---\n\n{body}" -# Skill definitions for config-driven generation -SKILL_DEFINITIONS = [ - { - 'name': 'write-flow', +# How each topic's guidance is assembled, for the chat and the CLI alike. +# +# `parts` are shared: the topic's chat helper (`chat_helper`, emitted into +# auto-generated/index.ts) and its CLI skill both concatenate them, in this +# order. A part is a file under base/ or languages/, or a token: {lang} (the +# language's own file), {sdk} (the SDK that language calls), {wac_sdk}, +# {openflow_schema}, {cli_commands}. A part written as ('chat', part) or +# ('cli', part) reaches that consumer only, and `cli_intro` heads the CLI skill. +# A sentence meant for one consumer inside a shared file is fenced instead +# (see `render_for`): adding a shared part here is what keeps the two in step. +TOPICS: dict[str, dict] = { + 'script': { + 'skill': 'write-script-{lang}', + 'chat_helper': 'getScriptPrompt', + 'cli_intro': 'script-cli.md', + 'parts': ['script-base.md', '{lang}', '{sdk}'], + }, + 'flow': { + 'skill': 'write-flow', 'description': 'MUST use when creating flows.', - 'content_key': 'flow', + 'chat_helper': 'getFlowPrompt', + 'cli_intro': 'flow-cli.md', + 'parts': ['flow-base.md', '{openflow_schema}'], }, - { - 'name': 'raw-app', + 'raw_app': { + 'skill': 'raw-app', 'description': 'MUST use when creating raw apps.', - 'content_key': 'raw_app', + 'chat_helper': 'getRawAppPrompt', + 'cli_intro': 'raw-app-cli.md', + # The CLI agent writes each runnable with the write-script- skill, + # which carries the SDK; the chat has no such skill to defer to. + 'parts': ['raw-app.md', ('chat', '{sdk}')], }, - { - 'name': 'triggers', + 'triggers': { + 'skill': 'triggers', 'description': 'MUST use when configuring triggers.', - 'content_key': 'triggers', - 'schema_types': [ - ('HttpTrigger', 'http_trigger'), - ('WebsocketTrigger', 'websocket_trigger'), - ('KafkaTrigger', 'kafka_trigger'), - ('NatsTrigger', 'nats_trigger'), - ('PostgresTrigger', 'postgres_trigger'), - ('MqttTrigger', 'mqtt_trigger'), - ('AmqpTrigger', 'amqp_trigger'), - ('SqsTrigger', 'sqs_trigger'), - ('GcpTrigger', 'gcp_trigger'), - ('AzureTrigger', 'azure_trigger'), - ('EmailTrigger', 'email_trigger'), - ], + 'parts': [('cli', 'triggers.md')], }, - { - 'name': 'schedules', + 'schedules': { + 'skill': 'schedules', 'description': 'MUST use when configuring schedules.', - 'content_key': 'schedules', - 'schema_types': [('Schedule', 'schedule')], + 'parts': [('cli', 'schedules.md')], }, - { - 'name': 'resources', + 'resources': { + 'skill': 'resources', 'description': 'MUST use when managing resources.', - 'content_key': 'resources', + 'chat_helper': 'getResourcePrompt', + 'parts': ['resources.md'], }, - { - 'name': 'write-workflow-as-code', + 'workflow_as_code': { + 'skill': 'write-workflow-as-code', 'description': 'MUST use when writing or modifying Windmill Workflow-as-Code scripts using workflow, task, step, sleep, approvals, taskScript, taskFlow, task_script, or task_flow.', - 'content_key': 'workflow_as_code', - 'intro_key': 'wac_cli', - 'sdk_content_key': 'wac', + 'chat_helper': 'getWorkflowAsCodePrompt', + 'cli_intro': 'script-cli.md', + 'parts': ['workflow-as-code.md', '{wac_sdk}'], }, - { - 'name': 'cli-commands', + 'pipeline': { + 'skill': 'write-pipeline', + 'description': 'MUST use when creating or modifying a data pipeline, a set of scripts marked `pipeline` and wired together by `on` / `materialize` annotations.', + 'chat_helper': 'getPipelinePrompt', + 'parts': ['pipeline-base.md'], + }, + 'cli_commands': { + 'skill': 'cli-commands', 'description': 'MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`.', - 'content_key': 'cli_commands', + 'parts': [('cli', '{cli_commands}')], }, - { - 'name': 'preview', + 'preview': { + 'skill': 'preview', 'description': 'MUST use when opening the Windmill dev page / visual preview of a flow, script, or app. Triggers on words like preview, open, navigate to, visualize, see the flow/app/script, and after writing a flow/script/app for visual verification.', - 'content_key': 'preview', + 'parts': [('cli', 'preview.md')], }, -] +} + +# The prompts.ts constant each shared file is exported as (chat render). +CHAT_CONSTANTS = { + 'script-base.md': 'SCRIPT_BASE', + 'flow-base.md': 'FLOW_BASE', + 'resources.md': 'RESOURCES_BASE', + 'raw-app.md': 'RAW_APP_BASE', + 'pipeline-base.md': 'PIPELINE_BASE', + 'workflow-as-code.md': 'WORKFLOW_AS_CODE_BASE', + 'flow-chat-special-modules.md': 'FLOW_CHAT_SPECIAL_MODULES', +} + + +def topic_parts(topic: str, consumer: str) -> list[str]: + """The parts of `topic` that reach `consumer`, in order.""" + parts = [] + for part in TOPICS[topic]['parts']: + if isinstance(part, tuple): + only, part = part + if only != consumer: + continue + parts.append(part) + return parts + + +def read_prompt_source(name: str, consumer: str) -> str: + """A base/ or languages/ markdown file, rendered for `consumer`.""" + for directory in (SCRIPT_DIR / "base", SCRIPT_DIR / "languages"): + path = directory / name + if path.exists(): + return render_for(path.read_text(), consumer, str(path.relative_to(SCRIPT_DIR))) + raise FileNotFoundError(f"No prompt source named {name} under base/ or languages/") + + +def ts_topic_return(topic: str) -> str: + """The `return [...]` of a chat helper: the topic's chat parts as TS expressions. + + `langPrompt` and `sdkPrompt` are locals every helper that uses those tokens + declares. + """ + token_exprs = { + '{lang}': 'langPrompt', + '{sdk}': 'sdkPrompt', + '{wac_sdk}': 'sdkPrompt', + '{openflow_schema}': 'prompts.OPENFLOW_SCHEMA', + } + exprs = [] + for part in topic_parts(topic, 'chat'): + if part in token_exprs: + exprs.append(token_exprs[part]) + elif part in CHAT_CONSTANTS: + exprs.append(f"prompts.{CHAT_CONSTANTS[part]}") + else: + raise ValueError(f"Topic '{topic}' part '{part}' has no chat export") + body = ",\n ".join(exprs) + return f"return [\n {body}\n ].filter(Boolean).join('\\n\\n');" + + +def ts_string_list(values: list[str]) -> str: + return "[" + ", ".join(f"'{v}'" for v in values) + "]" + + +def skill_descriptions() -> dict[str, str]: + """Map every skill name to its user-facing description.""" + desc_map = {} + for topic in TOPICS.values(): + if '{lang}' in topic['skill']: + for lang_key, metadata in LANGUAGE_METADATA.items(): + desc_map[topic['skill'].format(lang=lang_key)] = metadata['description'] + else: + desc_map[topic['skill']] = topic['description'] + return desc_map def generate_skills( @@ -1709,153 +1787,55 @@ def generate_skills( py_sdk_md: str, wac_ts_md: str, wac_py_md: str, - flow_cli: str, - flow_base: str, openflow_content: str, cli_commands: str, - cli_schemas: dict[str, dict] | None = None ): - """Generate individual skill files for Claude Code.""" + """Generate one CLI skill file per topic (and per language for scripts).""" print("Generating skill files...") - - cli_schemas = cli_schemas or {} - - # Ensure skills directory exists OUTPUT_SKILLS_DIR.mkdir(parents=True, exist_ok=True) + descriptions = skill_descriptions() - # Read base files for additional skills. - # Note: raw-app.md is the chat-relevant authoring guide. The CLI workflow - # (wmill app new wizard, on-disk layout, sql_to_apply/, CLI commands) lives - # in raw-app-cli.md. Concatenated here for the skill so CLI users see CLI - # guidance first, then the platform shape. - base_dir = SCRIPT_DIR / "base" - raw_app_cli_md = read_markdown_file(base_dir / "raw-app-cli.md") - raw_app_authoring_md = read_markdown_file(base_dir / "raw-app.md") - base_content = { - 'flow': f"{flow_cli}\n\n{flow_base}\n\n{openflow_content}", - 'raw_app': f"{raw_app_cli_md}\n\n{raw_app_authoring_md}", - 'triggers': read_markdown_file(base_dir / "triggers.md"), - 'schedules': read_markdown_file(base_dir / "schedules.md"), - 'resources': read_markdown_file(base_dir / "resources.md"), - 'workflow_as_code': read_markdown_file(base_dir / "workflow-as-code.md"), - 'cli_commands': cli_commands, - 'preview': read_markdown_file(base_dir / "preview.md"), - } + def resolve(part: str, lang: str | None) -> str: + if part == '{lang}': + return render_for(languages[lang], 'cli', f"languages/{lang}.md") + if part == '{sdk}': + if lang in TS_SDK_LANGUAGES: + return ts_sdk_md + if lang in PY_SDK_LANGUAGES: + return py_sdk_md + return '' + if part == '{wac_sdk}': + return "\n\n".join(filter(None, [wac_ts_md, wac_py_md])) + if part == '{openflow_schema}': + return openflow_content + if part == '{cli_commands}': + return cli_commands + return read_prompt_source(part, 'cli') - # CLI intro for script skills - script_cli_intro = """## CLI Commands - -Place scripts in a folder. - -After writing, tell the user which command fits what they want to do: - -- `wmill script preview ` — **default when iterating on a local script.** Runs the local file without deploying. -- `wmill script run ` — runs the script **already deployed** in the workspace. Use only when the user explicitly wants to test the deployed version, not local edits. -- `wmill generate-metadata` — regenerate the local `.script.yaml` (input schema) and `.lock` (resolved dependencies) for scripts you changed, and refresh their content hashes in `wmill-lock.yaml`. Local files only — **not** a deploy. See "Keep metadata in sync" below. -- Deploy local changes to the workspace — via `git push` or `wmill sync push` depending on how the repo is wired (see the **Deploying** section in `AGENTS.wmill.md`). Only suggest/run a deploy when the user explicitly asks to deploy/publish/push — not when they say "run", "try", or "test". - -### Preview vs run — choose by intent, not habit - -If the user says "run the script", "try it", "test it", "does it work" while there are **local edits to the script file**, use `script preview`. Do NOT push the script to then `script run` it — pushing is a deploy, and deploying just to test overwrites the workspace version with untested changes. - -Only use `script run` when: -- The user explicitly says "run the deployed version" / "run what's on the server". -- There is no local script being edited (you're just invoking an existing script). - -Only use `sync push` when: -- The user explicitly asks to deploy, publish, push, or ship. -- The preview has already validated the change and the user wants it in the workspace. - -### Keep metadata in sync after editing - -`wmill-lock.yaml` tracks a content hash for each item. Editing a script's content — most importantly **adding or removing an import** or **changing `main`'s arguments** — invalidates that hash and leaves the `.lock`, the `.script.yaml` input schema, and the hash row out of date. Run `wmill generate-metadata` (scoped to what you touched) after such edits so the resolved lock, the auto-generated args UI (driven by `.script.yaml`), and `wmill-lock.yaml` all match the code. Leaving them stale produces spurious diffs in git-sync and CI. - -This only writes local files (it is **not** a deploy), but it re-resolves dependencies, so it can bump unpinned versions (the same as deploying from the UI; expected, not a bug). So by default offer it and run it once the user agrees, rather than running it silently after every edit — unless the project's `AGENTS.md` opts into running metadata automatically (see the "Keeping metadata in sync" preference there). Either way YOU run the command, not the user. After running it, diff the regenerated `.lock` / `.script.lock` files and tell the user which dependency versions changed (e.g. `requests 2.31.0 → 2.32.0`), so they can catch an unwanted bump before deploying — even under `Metadata: auto`, since it's information, not a confirmation gate. Pin versions in code to keep them fixed. - -With no path argument, `generate-metadata` regenerates only the items whose content hash drifted — not everything. Imports propagate: editing a script that others import marks every importer stale too, so a one-line change to a shared module can regenerate many locks (by design — their locks must reflect the imported code). If it touches more than you expect, run `wmill generate-metadata --dry-run` — it lists each stale item with a reason (`content changed` or `depends on `) without changing anything — then narrow with a path argument (`wmill generate-metadata f/foo`) or `--strict-folder-boundaries`. - -If the on-disk `.lock` and `.script.yaml` are already correct and only `wmill-lock.yaml` needs its hashes refreshed (hash drift, or bootstrapping missing entries), use `wmill generate-metadata rehash` — it re-records hashes from disk with no backend round-trip and no dependency changes. - -### After writing — offer to test, don't wait passively - -If the user hasn't already told you to run/test/preview the script, offer it as a one-sentence next step (e.g. "Want me to run `wmill script preview` with sample args?"). Do not present a multi-option menu. - -If the user already asked to test/run/try the script in their original request, skip the offer and just execute `wmill script preview -d ''` directly — pick plausible args from the script's declared parameters. The shape varies by language: `main(...)` for code languages, the SQL dialect's own placeholder syntax (`$1` for PostgreSQL, `?` for MySQL/Snowflake, `@P1` for MSSQL, `@name` for BigQuery, etc.), positional `$1`, `$2`, … for Bash, `param(...)` for PowerShell. - -`wmill script preview` does not deploy, but it still executes script code and may cause side effects; run it yourself when the user asked to test/preview (or after confirming that execution is intended). `wmill generate-metadata` does not deploy either — it only writes local files (locks, schemas, hashes) — but offer it before running (or run automatically if the project's `AGENTS.md` opts in), per "Keep metadata in sync" above. Deploying to the workspace (`git push` or `wmill sync push` depending on how the repo is wired — see the **Deploying** section) is the only step that mutates remote state — do it only when the user explicitly asks to deploy/publish/push. - -For a **visual** open-the-script-in-the-dev-page preview (rather than `script preview`'s run-and-print-result), use the `preview` skill. - -Use `wmill resource-type list --schema` to discover available resource types.""" - - wac_cli_intro = f"""{script_cli_intro} - -Workflow-as-Code files use the normal script CLI workflow. There are no separate WAC deploy commands.""" - - intro_content = { - 'wac_cli': wac_cli_intro, - } - - extra_sdk_content = { - 'wac': "\n\n".join(filter(None, [wac_ts_md, wac_py_md])), - } + def write_skill(topic: dict, topic_key: str, lang: str | None = None) -> str: + skill_name = topic['skill'].format(lang=lang) if lang else topic['skill'] + sections = [] + if topic.get('cli_intro'): + sections.append(read_prompt_source(topic['cli_intro'], 'cli')) + sections.extend(resolve(part, lang) for part in topic_parts(topic_key, 'cli')) + body = "\n\n".join(s.rstrip('\n') for s in sections if s.strip()) + "\n" + skill_dir = OUTPUT_SKILLS_DIR / skill_name + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text( + generate_skill_content(skill_name, descriptions[skill_name], body) + ) + return skill_name skills_generated = [] - - # Generate script skills for each language - for lang_key, lang_content in languages.items(): - if lang_key not in LANGUAGE_METADATA: - print(f" Warning: No metadata for language '{lang_key}', skipping") - continue - - metadata = LANGUAGE_METADATA[lang_key] - skill_name = f"write-script-{lang_key}" - skill_dir = OUTPUT_SKILLS_DIR / skill_name - skill_dir.mkdir(parents=True, exist_ok=True) - - # Determine which SDK to include - language_sdk_content = '' - if lang_key in TS_SDK_LANGUAGES: - language_sdk_content = ts_sdk_md - elif lang_key in PY_SDK_LANGUAGES: - language_sdk_content = py_sdk_md - - skill_content = generate_skill_content( - skill_name=skill_name, - description=metadata['description'], - intro=script_cli_intro, - content=lang_content, - sdk_content=language_sdk_content - ) - - (skill_dir / "SKILL.md").write_text(skill_content) - skills_generated.append(skill_name) - - # Generate other skills from definitions - # Note: Skills with schema_types (triggers, schedules) get base content only. - # Schemas are stored separately and combined at CLI init time. - for skill_def in SKILL_DEFINITIONS: - content = base_content.get(skill_def['content_key'], '') - if not content: - continue - - skill_name = skill_def['name'] - skill_dir = OUTPUT_SKILLS_DIR / skill_name - skill_dir.mkdir(parents=True, exist_ok=True) - - # Note: We no longer append schemas here. Skills with 'schema_types' - # will have schemas combined at CLI init time from SCHEMAS export. - - skill_content = generate_skill_content( - skill_name=skill_name, - description=skill_def['description'], - intro=intro_content.get(skill_def.get('intro_key', ''), ''), - content=content, - sdk_content=extra_sdk_content.get(skill_def.get('sdk_content_key', ''), '') - ) - - (skill_dir / "SKILL.md").write_text(skill_content) - skills_generated.append(skill_name) + for topic_key, topic in TOPICS.items(): + if '{lang}' in topic['skill']: + for lang_key in languages: + if lang_key not in LANGUAGE_METADATA: + print(f" Warning: No metadata for language '{lang_key}', skipping") + continue + skills_generated.append(write_skill(topic, topic_key, lang_key)) + else: + skills_generated.append(write_skill(topic, topic_key)) print(f" Generated {len(skills_generated)} skills") return skills_generated @@ -1879,7 +1859,7 @@ def generate_skills_ts_export(skills: list[str], schema_yaml_content: dict[str, ts += "export const SKILLS: SkillMetadata[] = [\n" - skill_desc_map = {s['name']: s['description'] for s in SKILL_DEFINITIONS} + skill_desc_map = skill_descriptions() for skill in skills: if skill.startswith('write-script-'): @@ -2102,22 +2082,6 @@ def render_agents_md_for_docs( return template.replace("${skillsReference}", skills_reference) -def build_skill_desc_map(skills: list[str]) -> dict[str, str]: - """Map each skill name to its user-facing description. - - Mirrors the logic in `generate_skills_ts_export`: language skills draw from - LANGUAGE_METADATA, everything else from SKILL_DEFINITIONS. - """ - desc_map = {s["name"]: s["description"] for s in SKILL_DEFINITIONS} - for skill in skills: - if skill.startswith("write-script-"): - lang_key = skill.replace("write-script-", "") - metadata = LANGUAGE_METADATA.get(lang_key) - if metadata: - desc_map[skill] = metadata["description"] - return desc_map - - def _looks_like_windmill_manifest(path: Path) -> bool: """Return True iff `path` is a JSON file whose top-level `name` is ours. @@ -2227,7 +2191,7 @@ def generate_context7_repo( _verify_context7_target(target_dir) clear_context7_dir(target_dir) - skill_desc_map = build_skill_desc_map(skills) + skill_desc_map = skill_descriptions() # AGENTS.md — the managed CLI guidance (what `wmill init` writes as # AGENTS.wmill.md locally). Kept under the `AGENTS.md` filename here to @@ -2515,19 +2479,19 @@ def main(): base_dir = SCRIPT_DIR / "base" languages_dir = SCRIPT_DIR / "languages" - script_base = read_markdown_file(base_dir / "script-base.md") - flow_base = read_markdown_file(base_dir / "flow-base.md") - resources_base = read_markdown_file(base_dir / "resources.md") - raw_app_base = read_markdown_file(base_dir / "raw-app.md") - pipeline_base = read_markdown_file(base_dir / "pipeline-base.md") - workflow_as_code_base = read_markdown_file(base_dir / "workflow-as-code.md") - flow_cli = read_markdown_file(base_dir / "flow-cli.md") - flow_chat_special_modules = read_markdown_file(base_dir / "flow-chat-special-modules.md") + # The chat's exports; `generate_skills` renders the same files for the CLI. + chat_bases = { + const: read_prompt_source(name, 'chat') for name, const in CHAT_CONSTANTS.items() + } - # Read language files + # Read language files (rendered per consumer where they are used) languages = {} for lang_file in sorted(languages_dir.glob("*.md")): languages[lang_file.stem] = lang_file.read_text() + chat_languages = { + name: render_for(content, 'chat', f"languages/{name}.md") + for name, content in languages.items() + } # Extract and generate CLI commands documentation print("Extracting CLI commands...") @@ -2583,13 +2547,7 @@ def main(): # Assemble prompts for export prompts = { # Base prompts - 'SCRIPT_BASE': script_base, - 'FLOW_BASE': flow_base, - 'RESOURCES_BASE': resources_base, - 'RAW_APP_BASE': raw_app_base, - 'PIPELINE_BASE': pipeline_base, - 'WORKFLOW_AS_CODE_BASE': workflow_as_code_base, - 'FLOW_CHAT_SPECIAL_MODULES': flow_chat_special_modules, + **chat_bases, # SDKs 'SDK_TYPESCRIPT': ts_sdk_md, @@ -2609,7 +2567,7 @@ def main(): } # Add language prompts - for lang_name, lang_content in languages.items(): + for lang_name, lang_content in chat_languages.items(): prompts[f'LANG_{lang_name.upper()}'] = lang_content # Generate TypeScript exports @@ -2618,15 +2576,15 @@ def main(): (OUTPUT_GENERATED_DIR / "prompts.d.ts").write_text(generate_ts_declarations(prompts)) # Generate complete script.md (all languages combined) - script_md_parts = [script_base] - for lang_name in sorted(languages.keys()): - script_md_parts.append(languages[lang_name]) + script_md_parts = [chat_bases['SCRIPT_BASE']] + for lang_name in sorted(chat_languages.keys()): + script_md_parts.append(chat_languages[lang_name]) script_md_parts.extend([ts_sdk_md, py_sdk_md]) script_md = "\n\n".join(filter(None, script_md_parts)) (OUTPUT_GENERATED_DIR / "script.md").write_text(script_md) # Generate complete flow.md - flow_md_parts = [flow_base, openflow_content] + flow_md_parts = [chat_bases['FLOW_BASE'], openflow_content] flow_md = "\n\n".join(filter(None, flow_md_parts)) (OUTPUT_GENERATED_DIR / "flow.md").write_text(flow_md) @@ -2638,10 +2596,10 @@ export * from './prompts'; import * as prompts from './prompts'; // Languages that use the TypeScript SDK -const TS_SDK_LANGUAGES = ['bun', 'deno', 'nativets', 'bunnative']; +const TS_SDK_LANGUAGES = __TS_SDK_LANGUAGES__; // Languages that use the Python SDK -const PY_SDK_LANGUAGES = ['python3']; +const PY_SDK_LANGUAGES = __PY_SDK_LANGUAGES__; // Languages that use the TypeScript Workflow-as-Code SDK const WAC_TS_SDK_LANGUAGES = ['bun']; @@ -2662,24 +2620,17 @@ export function getScriptPrompt(language: string): string { sdkPrompt = prompts.SDK_PYTHON; } - return [ - prompts.SCRIPT_BASE, - langPrompt, - sdkPrompt - ].filter(Boolean).join('\\n\\n'); + __RETURN_script__ } // Helper to combine prompts for flows export function getFlowPrompt(): string { - return [ - prompts.FLOW_BASE, - prompts.OPENFLOW_SCHEMA - ].filter(Boolean).join('\\n\\n'); + __RETURN_flow__ } // Helper for resource & variable authoring export function getResourcePrompt(): string { - return prompts.RESOURCES_BASE; + __RETURN_resources__ } // Helper for raw app authoring (chat consumers). Inline backend runnables are @@ -2691,15 +2642,12 @@ export function getRawAppPrompt(language?: string): string { ? prompts.SDK_PYTHON : prompts.SDK_TYPESCRIPT; - return [ - prompts.RAW_APP_BASE, - sdkPrompt - ].filter(Boolean).join('\\n\\n'); + __RETURN_raw_app__ } // Helper for data pipeline authoring (chat consumers) export function getPipelinePrompt(): string { - return prompts.PIPELINE_BASE; + __RETURN_pipeline__ } // Helper to get the datatable SQL SDK reference (wmill.datatable()). @@ -2741,12 +2689,20 @@ export function getWorkflowAsCodePrompt(language?: string): string { return ''; } - return [ - prompts.WORKFLOW_AS_CODE_BASE, - sdkPrompt - ].filter(Boolean).join('\\n\\n'); + __RETURN_workflow_as_code__ } """ + index_content = (index_content + .replace('__TS_SDK_LANGUAGES__', ts_string_list(TS_SDK_LANGUAGES)) + .replace('__PY_SDK_LANGUAGES__', ts_string_list(PY_SDK_LANGUAGES)) + ) + for topic_key, topic in TOPICS.items(): + if 'chat_helper' not in topic: + continue + marker = f"__RETURN_{topic_key}__" + if marker not in index_content or f"function {topic['chat_helper']}(" not in index_content: + raise ValueError(f"index.ts has no {topic['chat_helper']} returning {marker}") + index_content = index_content.replace(marker, ts_topic_return(topic_key)) (OUTPUT_GENERATED_DIR / "index.ts").write_text(index_content) index_dts_content = """export * from './prompts'; @@ -2768,11 +2724,8 @@ export declare function getWorkflowAsCodePrompt(language?: string): string; py_sdk_md=py_sdk_md, wac_ts_md=wac_ts_md, wac_py_md=wac_py_md, - flow_cli=flow_cli, - flow_base=flow_base, cli_commands=cli_commands, openflow_content=openflow_content, - cli_schemas=cli_schemas ) # Generate skills TypeScript export for CLI diff --git a/system_prompts/languages/bun.md b/system_prompts/languages/bun.md index 22aeb2fb74..13c79c768e 100644 --- a/system_prompts/languages/bun.md +++ b/system_prompts/languages/bun.md @@ -29,7 +29,9 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + Before using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`. + ## Imports @@ -52,7 +54,7 @@ import * as wmill from "windmill-client"; **Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you, so you don't hand-roll URLs or tokens. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method to use instead of guessing or falling back to `fetch`. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method to use instead of guessing or falling back to `fetch`. ## Preprocessor Scripts @@ -70,7 +72,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; diff --git a/system_prompts/languages/bunnative.md b/system_prompts/languages/bunnative.md index cdcbcc6908..365a4ad4a1 100644 --- a/system_prompts/languages/bunnative.md +++ b/system_prompts/languages/bunnative.md @@ -31,7 +31,9 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + Before using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`. + ## Imports @@ -49,7 +51,7 @@ export async function main(url: string) { `windmill-client` works on the native worker (its calls go over `fetch`), so use it as the **preferred way to talk to Windmill** — reading resources/variables/states, running scripts and flows, and the S3 helpers below (`loadS3File`, `loadS3FileStream`, `writeS3File`, `S3Object`). It handles auth, the workspace, and the base URL for you. Reserve raw `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method instead of hand-rolling a `fetch` against the Windmill API. ## Preprocessor Scripts @@ -68,7 +70,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; diff --git a/system_prompts/languages/deno.md b/system_prompts/languages/deno.md index fddd08efb2..15519e1911 100644 --- a/system_prompts/languages/deno.md +++ b/system_prompts/languages/deno.md @@ -31,7 +31,9 @@ export async function main(stripe: RT.Stripe) { Only use resource types if you need them to satisfy the instructions. Always use the RT namespace. + Before using a resource type, check the `rt.d.ts` file in the project root to see all available resource types and their fields. This file is generated by `wmill resource-type generate-namespace`. + ## Imports @@ -54,7 +56,7 @@ import * as wmill from "windmill-client"; **Prefer `windmill-client` over raw `fetch` for anything that talks to Windmill** — reading resources/variables/states, running scripts and flows, S3 object operations, etc. It handles auth, the workspace, and the base URL for you. Reserve `fetch` for calling *external* HTTP APIs that aren't Windmill. -The full `windmill-client` API reference (every exported function and its signature) is included in this skill below — consult it for the exact method instead of guessing or falling back to `fetch`. +The full `windmill-client` API reference (every exported function and its signature) is included below — consult it for the exact method instead of guessing or falling back to `fetch`. ## Preprocessor Scripts @@ -72,7 +74,9 @@ type Event = { | "postgres" | "sqs" | "mqtt" - | "gcp"; + | "amqp" + | "gcp" + | "azure"; body: any; headers: Record; query: Record; diff --git a/system_prompts/languages/python3.md b/system_prompts/languages/python3.md index d556e45868..be4a2da68e 100644 --- a/system_prompts/languages/python3.md +++ b/system_prompts/languages/python3.md @@ -81,7 +81,7 @@ For preprocessor scripts, the function should be named `preprocessor` and receiv from typing import TypedDict, Literal, Any class Event(TypedDict): - kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "gcp"] + kind: Literal["webhook", "http", "websocket", "kafka", "email", "nats", "postgres", "sqs", "mqtt", "amqp", "gcp", "azure"] body: Any headers: dict[str, str] query: dict[str, str] diff --git a/system_prompts/utils.py b/system_prompts/utils.py index 75a592ece5..41bfc67851 100644 --- a/system_prompts/utils.py +++ b/system_prompts/utils.py @@ -243,6 +243,59 @@ def read_markdown_file(path: Path) -> str: return '' +CONSUMERS = ('cli', 'chat') +_FENCE_RE = re.compile(r'^$') +# Any comment opening with `cli`/`chat`, or closing on `only`, is taken as a fence +# attempt: a misspelled one would otherwise pass through with the text it guards. +_FENCE_LIKE_RE = re.compile( + r')', re.IGNORECASE +) + + +def render_for(md: str, consumer: str, source: str = '') -> str: + """Render shared markdown for one consumer ('cli' or 'chat'). + + A block between `` and `` (or the + `chat-only` pair) is kept for that consumer and dropped for the other; the + fence lines themselves never reach either. Fences must sit on their own + line and cannot nest. A malformed fence raises rather than leaking a + sentence meant for one consumer into the other's prompt. + """ + if consumer not in CONSUMERS: + raise ValueError(f"Unknown consumer '{consumer}'") + out: list[str] = [] + open_block: str | None = None + # Dropping a block would otherwise leave the blank lines around it doubled. + swallow_blank = False + for n, line in enumerate(md.splitlines(keepends=True), 1): + stripped = line.strip() + fence = _FENCE_RE.match(stripped) + if fence is None and _FENCE_LIKE_RE.search(stripped): + raise ValueError(f"{source}:{n}: malformed fence '{stripped}'") + if fence: + closing, who = fence.group(1) == '/', fence.group(2) + if closing: + if open_block != who: + raise ValueError(f"{source}:{n}: '{stripped}' closes no open {who}-only block") + swallow_blank = open_block != consumer + open_block = None + else: + if open_block is not None: + raise ValueError(f"{source}:{n}: {who}-only block opened inside a {open_block}-only block") + open_block = who + continue + if open_block is not None and open_block != consumer: + continue + if swallow_blank and stripped == '' and (not out or out[-1].strip() == ''): + swallow_blank = False + continue + swallow_blank = False + out.append(line) + if open_block is not None: + raise ValueError(f"{source}: {open_block}-only block is never closed") + return ''.join(out) + + # ============================================================================= # Parsing Utilities # ============================================================================= From 9f40cdca6258a9218770ebe1e7b89179e7d6667f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Sep 2026 15:24:39 +0200 Subject: [PATCH 73/91] feat: add pull_batch to claim jobs for many waiting workers at once (#11350) * feat: add pull_batch to claim jobs for many waiting workers at once Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep jobs admitted by earlier batch passes when a re-pull fails Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep suspended flows first on every batch re-pull pass Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/tests/batch_pull.rs | 278 +++++++++++++++++++++ backend/windmill-common/src/worker.rs | 159 +++++++++---- backend/windmill-queue/src/jobs.rs | 331 ++++++++++++++++++++------ 3 files changed, 652 insertions(+), 116 deletions(-) create mode 100644 backend/tests/batch_pull.rs diff --git a/backend/tests/batch_pull.rs b/backend/tests/batch_pull.rs new file mode 100644 index 0000000000..184b85dd06 --- /dev/null +++ b/backend/tests/batch_pull.rs @@ -0,0 +1,278 @@ +//! Pins what `pull_batch` promises a caller serving many waiting workers from one claim: every +//! claimed job is marked running under exactly one of those workers, concurrent batches never +//! claim the same job, the queues are walked in the same order a single pull walks them, and +//! the batch peek keeps the ordered index scan the single pull relies on. + +use std::collections::HashSet; + +use serde_json::Value; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::worker::{make_batch_pull_query, PullQueue}; +use windmill_queue::{pull_batch, PulledJobResult}; + +const W_ID: &str = "test-workspace"; + +async fn queue_job(db: &Pool, tag: &str, suspend: Option) -> anyhow::Result { + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, tag, args, visible_to_owner) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', 'noop', $3, \ + '{}', true)", + ) + .bind(id) + .bind(W_ID) + .bind(tag) + .execute(db) + .await?; + // A suspended flow sits running with `suspend_until` set; `suspend = 0` makes it resumable. + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, suspend, \ + suspend_until) \ + VALUES ($1, $2, now() - interval '1 second', $3, $4, COALESCE($5, 0), \ + CASE WHEN $5::int IS NULL THEN NULL ELSE now() + interval '1 day' END)", + ) + .bind(id) + .bind(W_ID) + .bind(suspend.is_some()) + .bind(tag) + .bind(suspend) + .execute(db) + .await?; + sqlx::query("INSERT INTO v2_job_runtime (id) VALUES ($1)") + .bind(id) + .execute(db) + .await?; + Ok(id) +} + +fn names(prefix: &str, n: usize) -> Vec { + (0..n).map(|i| format!("{prefix}-{i}")).collect() +} + +fn job_id(res: &PulledJobResult) -> Uuid { + res.job + .as_ref() + .expect("an admitted result carries its job") + .id +} + +#[sqlx::test(fixtures("base"))] +async fn batch_pull_assigns_each_job_to_one_waiting_worker( + db: Pool, +) -> anyhow::Result<()> { + for _ in 0..30 { + queue_job(&db, "batch", None).await?; + } + let groups = vec![vec!["batch".to_string()]]; + + // Three batches race over one queue; a name listed twice still gets one job. + let mut a = names("a", 8); + a.push("a-0".to_string()); + let (b, c) = (names("b", 8), names("c", 8)); + let (ra, rb, rc) = tokio::join!( + pull_batch(&db, &groups, &a, false), + pull_batch(&db, &groups, &b, false), + pull_batch(&db, &groups, &c, false), + ); + let claimed: Vec<(String, PulledJobResult)> = [ra?, rb?, rc?].into_iter().flatten().collect(); + assert_eq!( + claimed.len(), + 24, + "30 queued jobs cover all 24 distinct workers" + ); + let workers: HashSet<&String> = claimed.iter().map(|(w, _)| w).collect(); + let jobs: HashSet = claimed.iter().map(|(_, r)| job_id(r)).collect(); + assert_eq!(workers.len(), 24, "no worker is handed two jobs"); + assert_eq!(jobs.len(), 24, "no job is handed to two workers"); + + for (worker, res) in &claimed { + let (running, row_worker): (bool, Option) = + sqlx::query_as("SELECT running, worker FROM v2_job_queue WHERE id = $1") + .bind(job_id(res)) + .fetch_one(&db) + .await?; + assert!(running); + assert_eq!( + row_worker.as_ref(), + Some(worker), + "job is running under its worker" + ); + } + + // The result is the wire type a remote caller hands to its worker. + let (_, first) = &claimed[0]; + let round_trip: PulledJobResult = serde_json::from_str(&serde_json::to_string(first)?)?; + assert_eq!(job_id(&round_trip), job_id(first)); + + // Six jobs remain for ten workers: four go without. + let rest = pull_batch(&db, &groups, &names("d", 10), false).await?; + assert_eq!(rest.len(), 6); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn batch_pull_walks_suspended_then_priority_groups(db: Pool) -> anyhow::Result<()> { + let resumable = queue_job(&db, "low", Some(0)).await?; + let mut high = HashSet::new(); + for _ in 0..3 { + high.insert(queue_job(&db, "high", None).await?); + queue_job(&db, "low", None).await?; + } + let groups = vec![vec!["high".to_string()], vec!["low".to_string()]]; + + let claimed = pull_batch(&db, &groups, &names("w", 5), true).await?; + assert_eq!(claimed.len(), 5); + let suspended: Vec = claimed + .iter() + .filter(|(_, r)| r.suspended) + .map(|(_, r)| job_id(r)) + .collect(); + assert_eq!( + suspended, + vec![resumable], + "the resumable flow is claimed first" + ); + let ready: HashSet = claimed + .iter() + .filter(|(_, r)| !r.suspended) + .map(|(_, r)| job_id(r)) + .collect(); + assert!( + high.is_subset(&ready), + "the high group drains before the low one" + ); + assert_eq!(ready.len(), 4); + Ok(()) +} + +// A job over its concurrency limit must go back to the queue rather than sit claimed, and the +// worker it was claimed for is pulled for again in the same order: a resumable flow still wins +// over a ready job on the re-pull, as it does on every loop of a single pull. +#[cfg(feature = "enterprise")] +#[sqlx::test(fixtures("base"))] +async fn batch_pull_requeues_over_limit_job_and_serves_its_worker( + db: Pool, +) -> anyhow::Result<()> { + let limited = queue_job(&db, "batch", Some(0)).await?; + sqlx::query( + "UPDATE v2_job SET kind = 'script', runnable_path = 'f/test/limited', \ + concurrent_limit = 1, concurrency_time_window_s = 0 WHERE id = $1", + ) + .bind(limited) + .execute(&db) + .await?; + // Claimed first, so the bounce happens before the other jobs are reached. + sqlx::query("UPDATE v2_job_queue SET priority = 10 WHERE id = $1") + .bind(limited) + .execute(&db) + .await?; + sqlx::query("INSERT INTO concurrency_key (key, job_id) VALUES ('limited-key', $1)") + .bind(limited) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO concurrency_counter (concurrency_id, job_uuids) \ + VALUES ('limited-key', jsonb_build_object(gen_random_uuid()::text, '{}'::jsonb))", + ) + .execute(&db) + .await?; + let resumable = queue_job(&db, "batch", Some(0)).await?; + queue_job(&db, "batch", None).await?; + + let groups = vec![vec!["batch".to_string()]]; + let claimed = pull_batch(&db, &groups, &names("w", 1), true).await?; + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].0, "w-0"); + assert_eq!( + job_id(&claimed[0].1), + resumable, + "the worker is served the other resumable flow" + ); + assert!(claimed[0].1.suspended); + + let (running, rescheduled): (bool, bool) = + sqlx::query_as("SELECT running, scheduled_for > now() FROM v2_job_queue WHERE id = $1") + .bind(limited) + .fetch_one(&db) + .await?; + assert!(!running, "the over-limit job is back in the queue"); + assert!(rescheduled, "the over-limit job waits for a free slot"); + Ok(()) +} + +// Admission failing for one claimed job must hand it back to the queue: it is already marked +// running under its worker, which will never be told about it. +#[sqlx::test(fixtures("base"))] +async fn batch_pull_requeues_job_whose_admission_fails(db: Pool) -> anyhow::Result<()> { + let broken = queue_job(&db, "batch", None).await?; + // A settings handle with no `runnable_settings` row makes the settings lookup fail. + sqlx::query("UPDATE v2_job_queue SET runnable_settings_handle = -42 WHERE id = $1") + .bind(broken) + .execute(&db) + .await?; + + let groups = vec![vec!["batch".to_string()]]; + let claimed = pull_batch(&db, &groups, &names("w", 1), false).await?; + assert!(claimed.is_empty()); + + let (running, started): (bool, bool) = + sqlx::query_as("SELECT running, started_at IS NOT NULL FROM v2_job_queue WHERE id = $1") + .bind(broken) + .fetch_one(&db) + .await?; + assert!(!running && !started, "the job is back in the queue, not stranded"); + Ok(()) +} + +/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree. +fn nodes(plan: &Value, out: &mut Vec) { + out.push(plan.clone()); + for child in plan["Plans"].as_array().unwrap_or(&vec![]) { + nodes(child, out); + } +} + +// The batch LIMIT is a bind parameter, so a cached generic plan cannot see it and the planner +// falls back to guessing a fraction of the matching rows. On a deep backlog, a guess that tips +// it into a bitmap scan plus sort reads every queued job of the tag on each pull. +#[sqlx::test(fixtures("base"))] +async fn batch_pull_generic_plan_walks_the_queue_index(db: Pool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag) + SELECT gen_random_uuid(), 'test-workspace', now() - make_interval(secs => i % 1000), + false, CASE WHEN i % 10 = 0 THEN 'python3' ELSE 'deno' END + FROM generate_series(1, 200000) i", + ) + .execute(&db) + .await?; + sqlx::query("ANALYZE v2_job_queue").execute(&db).await?; + + let mut conn = db.acquire().await?; + sqlx::query("SET plan_cache_mode = force_generic_plan") + .execute(&mut *conn) + .await?; + let query = make_batch_pull_query(&["python3".to_string()], PullQueue::Ready); + sqlx::query(&format!("PREPARE batch_pull(text[]) AS {query}")) + .execute(&mut *conn) + .await?; + let explained: Value = + sqlx::query_scalar("EXPLAIN (FORMAT JSON) EXECUTE batch_pull(ARRAY['w-0', 'w-1'])") + .fetch_one(&mut *conn) + .await?; + + let mut all = vec![]; + nodes(&explained[0]["Plan"], &mut all); + let pretty = serde_json::to_string_pretty(&explained)?; + assert!( + all.iter() + .any(|n| n["Node Type"] == "Index Scan" && n["Index Name"] == "queue_sort_v2"), + "batch peek does not walk queue_sort_v2 in order:\n{pretty}" + ); + assert!( + !all.iter().any(|n| n["Node Type"] == "Sort"), + "batch peek sorts the backlog:\n{pretty}" + ); + Ok(()) +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index d480a89851..449fbd7606 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -774,6 +774,60 @@ impl From<&Pool> for Connection { } } +/// The columns every pull returns, read by name into `windmill_queue::PulledJob`. Expects `q` +/// (the claimed `v2_job_queue` rows), `j` (their `v2_job` rows), `f`, `p` and `pj` in scope. +const PULLED_JOB_COLUMNS: &str = "j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for, + j.runnable_id, j.runnable_path, j.args, q.canceled_by, + q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, + f.flow_status, j.script_lang, + j.same_worker, j.pre_run_error, j.visible_to_owner, + j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job, + j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow, + j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path, + COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, + p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email"; + +fn sql_tag_list(tags: &[String]) -> String { + tags.iter() + .map(|x| format!("'{}'", x.replace('\'', "''"))) + .join(", ") +} + +/// Selects the next runnable jobs for `tags`, highest priority first. `exclude_overloaded` names +/// the text[] bind parameter holding workspaces that workspace fairness currently caps. +fn ready_peek(tags: &[String], exclude_overloaded: Option<&str>, limit: &str) -> String { + let fairness = exclude_overloaded + .map(|param| format!("\n AND workspace_id <> ALL({param}::text[])")) + .unwrap_or_default(); + format!( + "SELECT id + FROM v2_job_queue + WHERE running = false AND tag IN ({}) AND scheduled_for <= now(){fairness} + ORDER BY priority DESC NULLS LAST, scheduled_for + FOR UPDATE SKIP LOCKED + LIMIT {limit}", + sql_tag_list(tags) + ) +} + +// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable +// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in +// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the +// test silently reverts to a heap filter over every suspended row on every worker poll. +fn suspended_peek(tags: &[String], limit: &str) -> String { + format!( + "SELECT id + FROM v2_job_queue + WHERE suspend_until IS NOT NULL + AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() + AND tag IN ({}) + ORDER BY priority DESC NULLS LAST, created_at + FOR UPDATE SKIP LOCKED + LIMIT {limit}", + sql_tag_list(tags) + ) +} + fn format_pull_query(peek: String) -> String { let r = format!( "WITH peek AS ( @@ -803,16 +857,7 @@ fn format_pull_query(peek: String) -> String { raw_flow, script_entrypoint_override, preprocessed FROM v2_job WHERE id = (SELECT id FROM peek) - ) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, started_at, scheduled_for, - j.runnable_id, j.runnable_path, j.args, canceled_by, - canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, - flow_status, j.script_lang, - j.same_worker, j.pre_run_error, j.visible_to_owner, - j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job, - j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow, - j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path, - COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, - p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email + ) SELECT {PULLED_JOB_COLUMNS} FROM q, j LEFT JOIN v2_job_status f USING (id) LEFT JOIN job_perms p ON p.job_id = j.id @@ -824,22 +869,63 @@ fn format_pull_query(peek: String) -> String { r } -// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable -// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in -// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the -// test silently reverts to a heap filter over every suspended row on every worker poll. +/// Claims up to `cardinality($1)` jobs in one statement and assigns the i-th claimed job to the +/// worker named `$1[i]`, so each row is marked running under the worker that will run it. +/// Returns the `PULLED_JOB_COLUMNS` plus `assigned_worker`. +fn format_batch_pull_query(peek: String) -> String { + format!( + "WITH peek AS MATERIALIZED ( + {peek} + ), assign AS ( + SELECT id, ($1::text[])[row_number() OVER ()] AS assigned_worker FROM peek + ), q AS ( + UPDATE v2_job_queue SET + running = true, + started_at = coalesce(started_at, now()), + suspend_until = null, + worker = assign.assigned_worker + FROM assign + WHERE v2_job_queue.id = assign.id + RETURNING + v2_job_queue.id, started_at, scheduled_for, + canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle + ), r AS ( + UPDATE v2_job_runtime SET + ping = now() + WHERE id = ANY(ARRAY(SELECT id FROM q)) + ) SELECT {PULLED_JOB_COLUMNS}, q.worker AS assigned_worker + FROM q + JOIN v2_job j ON j.id = q.id + LEFT JOIN v2_job_status f ON f.id = q.id + LEFT JOIN job_perms p ON p.job_id = q.id + LEFT JOIN v2_job pj ON j.parent_job = pj.id" + ) +} + +/// Which part of the queue a pull claims from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PullQueue { + /// Jobs ready to start. + Ready, + /// Jobs ready to start, excluding the workspaces bound as `$2::text[]`. + ReadyFairness, + /// Suspended flows whose resume condition is met. + Suspended, +} + +/// Batch variant of the pull queries: binds the claiming workers' names as `$1::text[]`, which +/// must be distinct, and claims at most one job per name. +pub fn make_batch_pull_query(tags: &[String], queue: PullQueue) -> String { + const LIMIT: &str = "cardinality($1::text[])"; + format_batch_pull_query(match queue { + PullQueue::Ready => ready_peek(tags, None, LIMIT), + PullQueue::ReadyFairness => ready_peek(tags, Some("$2"), LIMIT), + PullQueue::Suspended => suspended_peek(tags, LIMIT), + }) +} + pub fn make_suspended_pull_query(tags: &[String]) -> String { - format_pull_query(format!( - "SELECT id - FROM v2_job_queue - WHERE suspend_until IS NOT NULL - AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() - AND tag IN ({}) - ORDER BY priority DESC NULLS LAST, created_at - FOR UPDATE SKIP LOCKED - LIMIT 1", - tags.iter().map(|x| format!("'{x}'")).join(", ") - )) + format_pull_query(suspended_peek(tags, "1")) } // pub async fn make_suspended pub async fn store_suspended_pull_query(wc: &WorkerConfig) { @@ -852,16 +938,7 @@ pub async fn store_suspended_pull_query(wc: &WorkerConfig) { } pub fn make_pull_query(tags: &[String]) -> String { - let query = format_pull_query(format!( - "SELECT id - FROM v2_job_queue - WHERE running = false AND tag IN ({}) AND scheduled_for <= now() - ORDER BY priority DESC NULLS LAST, scheduled_for - FOR UPDATE SKIP LOCKED - LIMIT 1", - tags.iter().map(|x| format!("'{x}'")).join(", ") - )); - query + format_pull_query(ready_peek(tags, None, "1")) } // Variant of `make_pull_query` that additionally excludes jobs whose workspace_id is in the @@ -872,17 +949,7 @@ pub fn make_pull_query(tags: &[String]) -> String { // `pub(crate)` because only `store_pull_query` consumes it; the resulting query string is what // crosses crate boundaries via `WORKER_PULL_QUERIES_FAIRNESS`. pub(crate) fn make_pull_query_fairness(tags: &[String]) -> String { - let query = format_pull_query(format!( - "SELECT id - FROM v2_job_queue - WHERE running = false AND tag IN ({}) AND scheduled_for <= now() - AND workspace_id <> ALL($2::text[]) - ORDER BY priority DESC NULLS LAST, scheduled_for - FOR UPDATE SKIP LOCKED - LIMIT 1", - tags.iter().map(|x| format!("'{x}'")).join(", ") - )); - query + format_pull_query(ready_peek(tags, Some("$2"), "1")) } pub async fn store_pull_query(wc: &WorkerConfig) { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 7b552fb02c..8f65638790 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -78,9 +78,9 @@ use windmill_common::{ users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, worker::{ - to_raw_value, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, PREVIEW_TAGS_OVERRIDE, - WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS, WORKER_SUSPENDED_PULL_QUERY, - WORKSPACE_FAIRNESS_OVERLOADED, + make_batch_pull_query, to_raw_value, PullQueue, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, + PREVIEW_TAGS_OVERRIDE, WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS, + WORKER_SUSPENDED_PULL_QUERY, WORKSPACE_FAIRNESS_OVERLOADED, }, DB, METRICS_ENABLED, }; @@ -3570,7 +3570,7 @@ impl MiniPulledJob { } } -#[derive(sqlx::FromRow, Debug, Clone)] +#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)] pub struct PulledJob { #[sqlx(flatten)] pub job: MiniPulledJob, @@ -3837,7 +3837,7 @@ pub async fn get_queued_job_v2<'c>( Ok(job) } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct PulledJobResult { pub job: Option, pub suspended: bool, @@ -4228,6 +4228,13 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { Ok(()) } +// Cap to bound DB work per pull cycle. Each iteration on an over-limit job runs +// the full apply_concurrency_limit query stack (~7 queries), so without a tight +// cap a single pull() can issue thousands of queries when the queue is full of +// jobs sharing one over-limit concurrency key. Capping at 10 lets workers skip +// a few stale jobs in healthy conditions while preventing storm amplification. +const PULL_LOOP_LIMIT: i32 = 10; + // TODO: Factorize /// Pull the job from queue pub async fn pull( @@ -4242,14 +4249,9 @@ pub async fn pull( let mut pull_loop_count = 0; loop { pull_loop_count += 1; - // Cap to bound DB work per pull cycle. Each iteration on an over-limit job runs - // the full apply_concurrency_limit query stack (~7 queries), so without a tight - // cap a single pull() can issue thousands of queries when the queue is full of - // jobs sharing one over-limit concurrency key. Capping at 10 lets workers skip - // a few stale jobs in healthy conditions while preventing storm amplification. - if pull_loop_count > 10 { + if pull_loop_count > PULL_LOOP_LIMIT { tracing::warn!( - "Pull job loop count exceeded 10, backing off (likely concurrency re-queue storm)" + "Pull job loop count exceeded {PULL_LOOP_LIMIT}, backing off (likely concurrency re-queue storm)" ); return Ok(PulledJobResult { job: None, @@ -4389,68 +4391,257 @@ pub async fn pull( }); }; - let concurrency_settings = windmill_common::runnable_settings::prefetch_cached_from_handle( - job.runnable_settings_handle, - db, - ) - .await? - .1 - .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); - - let has_concurent_limit = - has_active_concurrency_limit(concurrency_settings.concurrent_limit); - - #[cfg(not(feature = "enterprise"))] - if has_concurent_limit && !job.is_dependency() { - tracing::error!("Concurrent limits are an EE feature only, ignoring constraints") - } - - #[cfg(not(feature = "enterprise"))] - let has_concurent_limit = job.is_dependency() - && has_active_concurrency_limit(job.concurrent_limit) - && cfg!(feature = "private") - && !*WMDEBUG_NO_DEBOUNCING; - // if we don't have private flag, we don't have concurrency limit - - // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue - let pulled_job = job; - if pulled_job.runnable_path.is_none() - || !has_concurent_limit - || pulled_job.canceled_by.is_some() - { - #[cfg(feature = "prometheus")] - if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { - QUEUE_PULL_COUNT.inc(); - } - otel_incr_queue_pull_count(); - return Ok(PulledJobResult { - job: Some(pulled_job), - suspended, - missing_concurrency_key: false, - error_while_preprocessing: None, - }); - } - - #[cfg(feature = "private")] - if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING) { - if let Some(pulled_job_res) = timeout( - Duration::from_secs(15), - crate::jobs_ee::apply_concurrency_limit( - db, - pull_loop_count, - suspended, - pulled_job, - &concurrency_settings, - ), - ) - .await?? - { - return Ok(pulled_job_res); - } + if let Some(pulled_job_res) = admit_pulled_job(db, job, suspended, pull_loop_count).await? { + return Ok(pulled_job_res); } } } +/// Applies a just-claimed job's concurrency limit, if it has one. `None` means the job was over +/// its limit and has been re-queued for later, so the claiming worker should pull again. +async fn admit_pulled_job( + db: &Pool, + job: PulledJob, + suspended: bool, + pull_loop_count: i32, +) -> windmill_common::error::Result> { + let concurrency_settings = windmill_common::runnable_settings::prefetch_cached_from_handle( + job.runnable_settings_handle, + db, + ) + .await? + .1 + .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); + + let has_concurent_limit = has_active_concurrency_limit(concurrency_settings.concurrent_limit); + + #[cfg(not(feature = "enterprise"))] + if has_concurent_limit && !job.is_dependency() { + tracing::error!("Concurrent limits are an EE feature only, ignoring constraints") + } + + #[cfg(not(feature = "enterprise"))] + let has_concurent_limit = job.is_dependency() + && has_active_concurrency_limit(job.concurrent_limit) + && cfg!(feature = "private") + && !*WMDEBUG_NO_DEBOUNCING; + // if we don't have private flag, we don't have concurrency limit + + // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue + let pulled_job = job; + if pulled_job.runnable_path.is_none() + || !has_concurent_limit + || pulled_job.canceled_by.is_some() + { + #[cfg(feature = "prometheus")] + if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + QUEUE_PULL_COUNT.inc(); + } + otel_incr_queue_pull_count(); + return Ok(Some(PulledJobResult { + job: Some(pulled_job), + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + })); + } + + #[cfg(feature = "private")] + if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING) { + return Ok(timeout( + Duration::from_secs(15), + crate::jobs_ee::apply_concurrency_limit( + db, + pull_loop_count, + suspended, + pulled_job, + &concurrency_settings, + ), + ) + .await??); + } + #[cfg(not(feature = "private"))] + let _ = pull_loop_count; + Ok(None) +} + +/// Claims jobs for several waiting workers in one statement per queue walked, instead of one +/// pull per worker. Each distinct name in `worker_names` gets at most one job, marked running +/// under that name; a worker missing from the result found nothing to run. +/// +/// Walks the queues the way [`pull`] does for one worker: when `suspend_first`, suspended flows +/// ready to resume on any of the tags; then `tag_groups` from highest priority down, with the +/// workspace-fairness share of the workers first restricted to uncapped workspaces. A job over +/// its concurrency limit is re-queued for later and its worker is pulled for again. +/// +/// No authorization happens here: jobs from every workspace matching the tags are claimed and +/// returned with their args and permissions. Callers must authenticate each worker named and +/// pass only tags that worker is allowed to serve. +pub async fn pull_batch( + db: &Pool, + tag_groups: &[Vec], + worker_names: &[String], + suspend_first: bool, +) -> windmill_common::error::Result> { + let mut waiting: Vec = worker_names.iter().unique().cloned().collect(); + let mut admitted = vec![]; + for pull_loop_count in 1..=PULL_LOOP_LIMIT { + if waiting.is_empty() { + break; + } + let claimed = match claim_batch(db, tag_groups, &waiting, suspend_first).await + { + Ok(claimed) => claimed, + Err(e) if admitted.is_empty() => return Err(e), + // Jobs admitted by an earlier pass are running under their workers: return them. + Err(e) => { + tracing::error!( + "batch pull re-pull failed, returning the jobs admitted so far: {e:#}" + ); + break; + } + }; + let mut requeued = false; + for (worker_name, job, suspended) in claimed { + waiting.retain(|w| w != &worker_name); + let job_id = job.id; + match admit_pulled_job(db, job, suspended, pull_loop_count).await { + Ok(Some(res)) => admitted.push((worker_name, res)), + Ok(None) => { + waiting.push(worker_name); + requeued = true; + } + // The other claimed jobs are already running under their workers, so an error + // here must not abort the batch: hand this one back to the queue instead. + Err(e) => { + tracing::error!( + "error admitting batch-pulled job {job_id}, re-queuing it: {e:#}" + ); + if let Err(e) = sqlx::query( + "WITH ping AS ( + UPDATE v2_job_runtime SET ping = null WHERE id = $1 + ) + UPDATE v2_job_queue SET running = false, started_at = null + WHERE id = $1 AND worker = $2 AND running = true", + ) + .bind(job_id) + .bind(&worker_name) + .execute(db) + .await + { + tracing::error!("could not re-queue batch-pulled job {job_id}: {e:#}"); + } + } + } + } + if !requeued { + break; + } + if pull_loop_count == PULL_LOOP_LIMIT { + tracing::warn!( + "Batch pull loop count reached {PULL_LOOP_LIMIT}, backing off (likely concurrency re-queue storm)" + ); + } + } + Ok(admitted) +} + +/// One pass over the queues for `worker_names`, returning `(worker, job, suspended)` for each +/// job claimed. A failing statement ends the pass but keeps what was already claimed, since +/// those jobs are marked running and would otherwise be stranded until the zombie monitor. +async fn claim_batch( + db: &Pool, + tag_groups: &[Vec], + worker_names: &[String], + suspend_first: bool, +) -> windmill_common::error::Result> { + let mut claimed = vec![]; + let mut waiting = worker_names.to_vec(); + let pass: windmill_common::error::Result<()> = async { + let tags: Vec = tag_groups.iter().flatten().unique().cloned().collect(); + if suspend_first && !tags.is_empty() { + for (worker, job) in + claim_from_queue(db, &tags, PullQueue::Suspended, &[], &waiting).await? + { + waiting.retain(|w| w != &worker); + claimed.push((worker, job, true)); + } + } + + crate::workspace_fairness::maybe_refresh_overloaded(db); + let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full(); + if !overloaded.is_empty() { + // Same admission draw `pull` makes once per worker, made here once per waiting worker. + let mut fair: Vec = waiting + .iter() + .filter(|_| !crate::workspace_fairness::should_admit_capped()) + .cloned() + .collect(); + for tags in tag_groups.iter().filter(|t| !t.is_empty()) { + if fair.is_empty() { + break; + } + for (worker, job) in claim_from_queue( + db, + tags, + PullQueue::ReadyFairness, + overloaded.as_slice(), + &fair, + ) + .await? + { + fair.retain(|w| w != &worker); + waiting.retain(|w| w != &worker); + claimed.push((worker, job, false)); + } + } + } + + for tags in tag_groups.iter().filter(|t| !t.is_empty()) { + if waiting.is_empty() { + break; + } + for (worker, job) in claim_from_queue(db, tags, PullQueue::Ready, &[], &waiting).await? + { + waiting.retain(|w| w != &worker); + claimed.push((worker, job, false)); + } + } + Ok(()) + } + .await; + match pass { + Err(e) if claimed.is_empty() => Err(e), + Err(e) => { + tracing::error!( + "batch pull pass failed after claiming {} jobs: {e:#}", + claimed.len() + ); + Ok(claimed) + } + Ok(()) => Ok(claimed), + } +} + +async fn claim_from_queue( + db: &Pool, + tags: &[String], + queue: PullQueue, + overloaded: &[String], + worker_names: &[String], +) -> windmill_common::error::Result> { + use sqlx::{FromRow, Row}; + let query = make_batch_pull_query(tags, queue); + let mut query = sqlx::query(&query).bind(worker_names); + if queue == PullQueue::ReadyFairness { + query = query.bind(overloaded); + } + let rows = timeout(Duration::from_secs(15), query.fetch_all(db)).await??; + rows.iter() + .map(|row| Ok((row.try_get("assigned_worker")?, PulledJob::from_row(row)?))) + .collect() +} + async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( db: &Pool, suspend_first: bool, From 3936191f46367653179a0820bb8281e815a9dedb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Sep 2026 15:25:35 +0200 Subject: [PATCH 74/91] =?UTF-8?q?Revert=20"feat:=20add=20pull=5Fbatch=20to?= =?UTF-8?q?=20claim=20jobs=20for=20many=20waiting=20workers=20at=20once=20?= =?UTF-8?q?(=E2=80=A6"=20(#11412)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9f40cdca6258a9218770ebe1e7b89179e7d6667f. --- backend/tests/batch_pull.rs | 278 ----------------------- backend/windmill-common/src/worker.rs | 159 ++++--------- backend/windmill-queue/src/jobs.rs | 311 +++++--------------------- 3 files changed, 106 insertions(+), 642 deletions(-) delete mode 100644 backend/tests/batch_pull.rs diff --git a/backend/tests/batch_pull.rs b/backend/tests/batch_pull.rs deleted file mode 100644 index 184b85dd06..0000000000 --- a/backend/tests/batch_pull.rs +++ /dev/null @@ -1,278 +0,0 @@ -//! Pins what `pull_batch` promises a caller serving many waiting workers from one claim: every -//! claimed job is marked running under exactly one of those workers, concurrent batches never -//! claim the same job, the queues are walked in the same order a single pull walks them, and -//! the batch peek keeps the ordered index scan the single pull relies on. - -use std::collections::HashSet; - -use serde_json::Value; -use sqlx::{Pool, Postgres}; -use uuid::Uuid; -use windmill_common::worker::{make_batch_pull_query, PullQueue}; -use windmill_queue::{pull_batch, PulledJobResult}; - -const W_ID: &str = "test-workspace"; - -async fn queue_job(db: &Pool, tag: &str, suspend: Option) -> anyhow::Result { - let id = Uuid::new_v4(); - sqlx::query( - "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ - permissioned_as_email, kind, tag, args, visible_to_owner) \ - VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', 'noop', $3, \ - '{}', true)", - ) - .bind(id) - .bind(W_ID) - .bind(tag) - .execute(db) - .await?; - // A suspended flow sits running with `suspend_until` set; `suspend = 0` makes it resumable. - sqlx::query( - "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, suspend, \ - suspend_until) \ - VALUES ($1, $2, now() - interval '1 second', $3, $4, COALESCE($5, 0), \ - CASE WHEN $5::int IS NULL THEN NULL ELSE now() + interval '1 day' END)", - ) - .bind(id) - .bind(W_ID) - .bind(suspend.is_some()) - .bind(tag) - .bind(suspend) - .execute(db) - .await?; - sqlx::query("INSERT INTO v2_job_runtime (id) VALUES ($1)") - .bind(id) - .execute(db) - .await?; - Ok(id) -} - -fn names(prefix: &str, n: usize) -> Vec { - (0..n).map(|i| format!("{prefix}-{i}")).collect() -} - -fn job_id(res: &PulledJobResult) -> Uuid { - res.job - .as_ref() - .expect("an admitted result carries its job") - .id -} - -#[sqlx::test(fixtures("base"))] -async fn batch_pull_assigns_each_job_to_one_waiting_worker( - db: Pool, -) -> anyhow::Result<()> { - for _ in 0..30 { - queue_job(&db, "batch", None).await?; - } - let groups = vec![vec!["batch".to_string()]]; - - // Three batches race over one queue; a name listed twice still gets one job. - let mut a = names("a", 8); - a.push("a-0".to_string()); - let (b, c) = (names("b", 8), names("c", 8)); - let (ra, rb, rc) = tokio::join!( - pull_batch(&db, &groups, &a, false), - pull_batch(&db, &groups, &b, false), - pull_batch(&db, &groups, &c, false), - ); - let claimed: Vec<(String, PulledJobResult)> = [ra?, rb?, rc?].into_iter().flatten().collect(); - assert_eq!( - claimed.len(), - 24, - "30 queued jobs cover all 24 distinct workers" - ); - let workers: HashSet<&String> = claimed.iter().map(|(w, _)| w).collect(); - let jobs: HashSet = claimed.iter().map(|(_, r)| job_id(r)).collect(); - assert_eq!(workers.len(), 24, "no worker is handed two jobs"); - assert_eq!(jobs.len(), 24, "no job is handed to two workers"); - - for (worker, res) in &claimed { - let (running, row_worker): (bool, Option) = - sqlx::query_as("SELECT running, worker FROM v2_job_queue WHERE id = $1") - .bind(job_id(res)) - .fetch_one(&db) - .await?; - assert!(running); - assert_eq!( - row_worker.as_ref(), - Some(worker), - "job is running under its worker" - ); - } - - // The result is the wire type a remote caller hands to its worker. - let (_, first) = &claimed[0]; - let round_trip: PulledJobResult = serde_json::from_str(&serde_json::to_string(first)?)?; - assert_eq!(job_id(&round_trip), job_id(first)); - - // Six jobs remain for ten workers: four go without. - let rest = pull_batch(&db, &groups, &names("d", 10), false).await?; - assert_eq!(rest.len(), 6); - Ok(()) -} - -#[sqlx::test(fixtures("base"))] -async fn batch_pull_walks_suspended_then_priority_groups(db: Pool) -> anyhow::Result<()> { - let resumable = queue_job(&db, "low", Some(0)).await?; - let mut high = HashSet::new(); - for _ in 0..3 { - high.insert(queue_job(&db, "high", None).await?); - queue_job(&db, "low", None).await?; - } - let groups = vec![vec!["high".to_string()], vec!["low".to_string()]]; - - let claimed = pull_batch(&db, &groups, &names("w", 5), true).await?; - assert_eq!(claimed.len(), 5); - let suspended: Vec = claimed - .iter() - .filter(|(_, r)| r.suspended) - .map(|(_, r)| job_id(r)) - .collect(); - assert_eq!( - suspended, - vec![resumable], - "the resumable flow is claimed first" - ); - let ready: HashSet = claimed - .iter() - .filter(|(_, r)| !r.suspended) - .map(|(_, r)| job_id(r)) - .collect(); - assert!( - high.is_subset(&ready), - "the high group drains before the low one" - ); - assert_eq!(ready.len(), 4); - Ok(()) -} - -// A job over its concurrency limit must go back to the queue rather than sit claimed, and the -// worker it was claimed for is pulled for again in the same order: a resumable flow still wins -// over a ready job on the re-pull, as it does on every loop of a single pull. -#[cfg(feature = "enterprise")] -#[sqlx::test(fixtures("base"))] -async fn batch_pull_requeues_over_limit_job_and_serves_its_worker( - db: Pool, -) -> anyhow::Result<()> { - let limited = queue_job(&db, "batch", Some(0)).await?; - sqlx::query( - "UPDATE v2_job SET kind = 'script', runnable_path = 'f/test/limited', \ - concurrent_limit = 1, concurrency_time_window_s = 0 WHERE id = $1", - ) - .bind(limited) - .execute(&db) - .await?; - // Claimed first, so the bounce happens before the other jobs are reached. - sqlx::query("UPDATE v2_job_queue SET priority = 10 WHERE id = $1") - .bind(limited) - .execute(&db) - .await?; - sqlx::query("INSERT INTO concurrency_key (key, job_id) VALUES ('limited-key', $1)") - .bind(limited) - .execute(&db) - .await?; - sqlx::query( - "INSERT INTO concurrency_counter (concurrency_id, job_uuids) \ - VALUES ('limited-key', jsonb_build_object(gen_random_uuid()::text, '{}'::jsonb))", - ) - .execute(&db) - .await?; - let resumable = queue_job(&db, "batch", Some(0)).await?; - queue_job(&db, "batch", None).await?; - - let groups = vec![vec!["batch".to_string()]]; - let claimed = pull_batch(&db, &groups, &names("w", 1), true).await?; - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].0, "w-0"); - assert_eq!( - job_id(&claimed[0].1), - resumable, - "the worker is served the other resumable flow" - ); - assert!(claimed[0].1.suspended); - - let (running, rescheduled): (bool, bool) = - sqlx::query_as("SELECT running, scheduled_for > now() FROM v2_job_queue WHERE id = $1") - .bind(limited) - .fetch_one(&db) - .await?; - assert!(!running, "the over-limit job is back in the queue"); - assert!(rescheduled, "the over-limit job waits for a free slot"); - Ok(()) -} - -// Admission failing for one claimed job must hand it back to the queue: it is already marked -// running under its worker, which will never be told about it. -#[sqlx::test(fixtures("base"))] -async fn batch_pull_requeues_job_whose_admission_fails(db: Pool) -> anyhow::Result<()> { - let broken = queue_job(&db, "batch", None).await?; - // A settings handle with no `runnable_settings` row makes the settings lookup fail. - sqlx::query("UPDATE v2_job_queue SET runnable_settings_handle = -42 WHERE id = $1") - .bind(broken) - .execute(&db) - .await?; - - let groups = vec![vec!["batch".to_string()]]; - let claimed = pull_batch(&db, &groups, &names("w", 1), false).await?; - assert!(claimed.is_empty()); - - let (running, started): (bool, bool) = - sqlx::query_as("SELECT running, started_at IS NOT NULL FROM v2_job_queue WHERE id = $1") - .bind(broken) - .fetch_one(&db) - .await?; - assert!(!running && !started, "the job is back in the queue, not stranded"); - Ok(()) -} - -/// Depth-first walk of an `EXPLAIN (FORMAT JSON)` plan tree. -fn nodes(plan: &Value, out: &mut Vec) { - out.push(plan.clone()); - for child in plan["Plans"].as_array().unwrap_or(&vec![]) { - nodes(child, out); - } -} - -// The batch LIMIT is a bind parameter, so a cached generic plan cannot see it and the planner -// falls back to guessing a fraction of the matching rows. On a deep backlog, a guess that tips -// it into a bitmap scan plus sort reads every queued job of the tag on each pull. -#[sqlx::test(fixtures("base"))] -async fn batch_pull_generic_plan_walks_the_queue_index(db: Pool) -> anyhow::Result<()> { - sqlx::query( - "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag) - SELECT gen_random_uuid(), 'test-workspace', now() - make_interval(secs => i % 1000), - false, CASE WHEN i % 10 = 0 THEN 'python3' ELSE 'deno' END - FROM generate_series(1, 200000) i", - ) - .execute(&db) - .await?; - sqlx::query("ANALYZE v2_job_queue").execute(&db).await?; - - let mut conn = db.acquire().await?; - sqlx::query("SET plan_cache_mode = force_generic_plan") - .execute(&mut *conn) - .await?; - let query = make_batch_pull_query(&["python3".to_string()], PullQueue::Ready); - sqlx::query(&format!("PREPARE batch_pull(text[]) AS {query}")) - .execute(&mut *conn) - .await?; - let explained: Value = - sqlx::query_scalar("EXPLAIN (FORMAT JSON) EXECUTE batch_pull(ARRAY['w-0', 'w-1'])") - .fetch_one(&mut *conn) - .await?; - - let mut all = vec![]; - nodes(&explained[0]["Plan"], &mut all); - let pretty = serde_json::to_string_pretty(&explained)?; - assert!( - all.iter() - .any(|n| n["Node Type"] == "Index Scan" && n["Index Name"] == "queue_sort_v2"), - "batch peek does not walk queue_sort_v2 in order:\n{pretty}" - ); - assert!( - !all.iter().any(|n| n["Node Type"] == "Sort"), - "batch peek sorts the backlog:\n{pretty}" - ); - Ok(()) -} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 449fbd7606..d480a89851 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -774,60 +774,6 @@ impl From<&Pool> for Connection { } } -/// The columns every pull returns, read by name into `windmill_queue::PulledJob`. Expects `q` -/// (the claimed `v2_job_queue` rows), `j` (their `v2_job` rows), `f`, `p` and `pj` in scope. -const PULLED_JOB_COLUMNS: &str = "j.id, j.workspace_id, j.parent_job, j.created_by, q.started_at, q.scheduled_for, - j.runnable_id, j.runnable_path, j.args, q.canceled_by, - q.canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, - f.flow_status, j.script_lang, - j.same_worker, j.pre_run_error, j.visible_to_owner, - j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job, - j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow, - j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path, - COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, - p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email"; - -fn sql_tag_list(tags: &[String]) -> String { - tags.iter() - .map(|x| format!("'{}'", x.replace('\'', "''"))) - .join(", ") -} - -/// Selects the next runnable jobs for `tags`, highest priority first. `exclude_overloaded` names -/// the text[] bind parameter holding workspaces that workspace fairness currently caps. -fn ready_peek(tags: &[String], exclude_overloaded: Option<&str>, limit: &str) -> String { - let fairness = exclude_overloaded - .map(|param| format!("\n AND workspace_id <> ALL({param}::text[])")) - .unwrap_or_default(); - format!( - "SELECT id - FROM v2_job_queue - WHERE running = false AND tag IN ({}) AND scheduled_for <= now(){fairness} - ORDER BY priority DESC NULLS LAST, scheduled_for - FOR UPDATE SKIP LOCKED - LIMIT {limit}", - sql_tag_list(tags) - ) -} - -// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable -// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in -// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the -// test silently reverts to a heap filter over every suspended row on every worker poll. -fn suspended_peek(tags: &[String], limit: &str) -> String { - format!( - "SELECT id - FROM v2_job_queue - WHERE suspend_until IS NOT NULL - AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() - AND tag IN ({}) - ORDER BY priority DESC NULLS LAST, created_at - FOR UPDATE SKIP LOCKED - LIMIT {limit}", - sql_tag_list(tags) - ) -} - fn format_pull_query(peek: String) -> String { let r = format!( "WITH peek AS ( @@ -857,7 +803,16 @@ fn format_pull_query(peek: String) -> String { raw_flow, script_entrypoint_override, preprocessed FROM v2_job WHERE id = (SELECT id FROM peek) - ) SELECT {PULLED_JOB_COLUMNS} + ) SELECT j.id, j.workspace_id, j.parent_job, j.created_by, started_at, scheduled_for, + j.runnable_id, j.runnable_path, j.args, canceled_by, + canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, + flow_status, j.script_lang, + j.same_worker, j.pre_run_error, j.visible_to_owner, + j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job, + j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle, j.priority, j.raw_code, j.raw_lock, j.raw_flow, + j.script_entrypoint_override, j.preprocessed, COALESCE(pj.runnable_path, j.args->>'_FLOW_PATH') as parent_runnable_path, + COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, + p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email FROM q, j LEFT JOIN v2_job_status f USING (id) LEFT JOIN job_perms p ON p.job_id = j.id @@ -869,63 +824,22 @@ fn format_pull_query(peek: String) -> String { r } -/// Claims up to `cardinality($1)` jobs in one statement and assigns the i-th claimed job to the -/// worker named `$1[i]`, so each row is marked running under the worker that will run it. -/// Returns the `PULLED_JOB_COLUMNS` plus `assigned_worker`. -fn format_batch_pull_query(peek: String) -> String { - format!( - "WITH peek AS MATERIALIZED ( - {peek} - ), assign AS ( - SELECT id, ($1::text[])[row_number() OVER ()] AS assigned_worker FROM peek - ), q AS ( - UPDATE v2_job_queue SET - running = true, - started_at = coalesce(started_at, now()), - suspend_until = null, - worker = assign.assigned_worker - FROM assign - WHERE v2_job_queue.id = assign.id - RETURNING - v2_job_queue.id, started_at, scheduled_for, - canceled_by, canceled_reason, worker, cache_ignore_s3_path, runnable_settings_handle - ), r AS ( - UPDATE v2_job_runtime SET - ping = now() - WHERE id = ANY(ARRAY(SELECT id FROM q)) - ) SELECT {PULLED_JOB_COLUMNS}, q.worker AS assigned_worker - FROM q - JOIN v2_job j ON j.id = q.id - LEFT JOIN v2_job_status f ON f.id = q.id - LEFT JOIN job_perms p ON p.job_id = q.id - LEFT JOIN v2_job pj ON j.parent_job = pj.id" - ) -} - -/// Which part of the queue a pull claims from. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PullQueue { - /// Jobs ready to start. - Ready, - /// Jobs ready to start, excluding the workspaces bound as `$2::text[]`. - ReadyFairness, - /// Suspended flows whose resume condition is met. - Suspended, -} - -/// Batch variant of the pull queries: binds the claiming workers' names as `$1::text[]`, which -/// must be distinct, and claims at most one job per name. -pub fn make_batch_pull_query(tags: &[String], queue: PullQueue) -> String { - const LIMIT: &str = "cardinality($1::text[])"; - format_batch_pull_query(match queue { - PullQueue::Ready => ready_peek(tags, None, LIMIT), - PullQueue::ReadyFairness => ready_peek(tags, Some("$2"), LIMIT), - PullQueue::Suspended => suspended_peek(tags, LIMIT), - }) -} - +// The `CASE` is `suspend <= 0 OR suspend_until <= now()` written as one indexable +// expression, equivalent only under the `suspend_until IS NOT NULL` guard. It must stay in +// sync with `queue_suspended_v2` (migration 20260826202939): if it no longer matches, the +// test silently reverts to a heap filter over every suspended row on every worker poll. pub fn make_suspended_pull_query(tags: &[String]) -> String { - format_pull_query(suspended_peek(tags, "1")) + format_pull_query(format!( + "SELECT id + FROM v2_job_queue + WHERE suspend_until IS NOT NULL + AND (CASE WHEN suspend <= 0 THEN '-infinity'::timestamptz ELSE suspend_until END) <= now() + AND tag IN ({}) + ORDER BY priority DESC NULLS LAST, created_at + FOR UPDATE SKIP LOCKED + LIMIT 1", + tags.iter().map(|x| format!("'{x}'")).join(", ") + )) } // pub async fn make_suspended pub async fn store_suspended_pull_query(wc: &WorkerConfig) { @@ -938,7 +852,16 @@ pub async fn store_suspended_pull_query(wc: &WorkerConfig) { } pub fn make_pull_query(tags: &[String]) -> String { - format_pull_query(ready_peek(tags, None, "1")) + let query = format_pull_query(format!( + "SELECT id + FROM v2_job_queue + WHERE running = false AND tag IN ({}) AND scheduled_for <= now() + ORDER BY priority DESC NULLS LAST, scheduled_for + FOR UPDATE SKIP LOCKED + LIMIT 1", + tags.iter().map(|x| format!("'{x}'")).join(", ") + )); + query } // Variant of `make_pull_query` that additionally excludes jobs whose workspace_id is in the @@ -949,7 +872,17 @@ pub fn make_pull_query(tags: &[String]) -> String { // `pub(crate)` because only `store_pull_query` consumes it; the resulting query string is what // crosses crate boundaries via `WORKER_PULL_QUERIES_FAIRNESS`. pub(crate) fn make_pull_query_fairness(tags: &[String]) -> String { - format_pull_query(ready_peek(tags, Some("$2"), "1")) + let query = format_pull_query(format!( + "SELECT id + FROM v2_job_queue + WHERE running = false AND tag IN ({}) AND scheduled_for <= now() + AND workspace_id <> ALL($2::text[]) + ORDER BY priority DESC NULLS LAST, scheduled_for + FOR UPDATE SKIP LOCKED + LIMIT 1", + tags.iter().map(|x| format!("'{x}'")).join(", ") + )); + query } pub async fn store_pull_query(wc: &WorkerConfig) { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 8f65638790..7b552fb02c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -78,9 +78,9 @@ use windmill_common::{ users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, worker::{ - make_batch_pull_query, to_raw_value, PullQueue, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, - PREVIEW_TAGS_OVERRIDE, WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS, - WORKER_SUSPENDED_PULL_QUERY, WORKSPACE_FAIRNESS_OVERLOADED, + to_raw_value, CLOUD_HOSTED, DISABLE_FLOW_SCRIPT, NO_LOGS, PREVIEW_TAGS_OVERRIDE, + WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS, WORKER_SUSPENDED_PULL_QUERY, + WORKSPACE_FAIRNESS_OVERLOADED, }, DB, METRICS_ENABLED, }; @@ -3570,7 +3570,7 @@ impl MiniPulledJob { } } -#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)] +#[derive(sqlx::FromRow, Debug, Clone)] pub struct PulledJob { #[sqlx(flatten)] pub job: MiniPulledJob, @@ -3837,7 +3837,7 @@ pub async fn get_queued_job_v2<'c>( Ok(job) } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug)] pub struct PulledJobResult { pub job: Option, pub suspended: bool, @@ -4228,13 +4228,6 @@ async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> { Ok(()) } -// Cap to bound DB work per pull cycle. Each iteration on an over-limit job runs -// the full apply_concurrency_limit query stack (~7 queries), so without a tight -// cap a single pull() can issue thousands of queries when the queue is full of -// jobs sharing one over-limit concurrency key. Capping at 10 lets workers skip -// a few stale jobs in healthy conditions while preventing storm amplification. -const PULL_LOOP_LIMIT: i32 = 10; - // TODO: Factorize /// Pull the job from queue pub async fn pull( @@ -4249,9 +4242,14 @@ pub async fn pull( let mut pull_loop_count = 0; loop { pull_loop_count += 1; - if pull_loop_count > PULL_LOOP_LIMIT { + // Cap to bound DB work per pull cycle. Each iteration on an over-limit job runs + // the full apply_concurrency_limit query stack (~7 queries), so without a tight + // cap a single pull() can issue thousands of queries when the queue is full of + // jobs sharing one over-limit concurrency key. Capping at 10 lets workers skip + // a few stale jobs in healthy conditions while preventing storm amplification. + if pull_loop_count > 10 { tracing::warn!( - "Pull job loop count exceeded {PULL_LOOP_LIMIT}, backing off (likely concurrency re-queue storm)" + "Pull job loop count exceeded 10, backing off (likely concurrency re-queue storm)" ); return Ok(PulledJobResult { job: None, @@ -4391,255 +4389,66 @@ pub async fn pull( }); }; - if let Some(pulled_job_res) = admit_pulled_job(db, job, suspended, pull_loop_count).await? { - return Ok(pulled_job_res); - } - } -} - -/// Applies a just-claimed job's concurrency limit, if it has one. `None` means the job was over -/// its limit and has been re-queued for later, so the claiming worker should pull again. -async fn admit_pulled_job( - db: &Pool, - job: PulledJob, - suspended: bool, - pull_loop_count: i32, -) -> windmill_common::error::Result> { - let concurrency_settings = windmill_common::runnable_settings::prefetch_cached_from_handle( - job.runnable_settings_handle, - db, - ) - .await? - .1 - .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); - - let has_concurent_limit = has_active_concurrency_limit(concurrency_settings.concurrent_limit); - - #[cfg(not(feature = "enterprise"))] - if has_concurent_limit && !job.is_dependency() { - tracing::error!("Concurrent limits are an EE feature only, ignoring constraints") - } - - #[cfg(not(feature = "enterprise"))] - let has_concurent_limit = job.is_dependency() - && has_active_concurrency_limit(job.concurrent_limit) - && cfg!(feature = "private") - && !*WMDEBUG_NO_DEBOUNCING; - // if we don't have private flag, we don't have concurrency limit - - // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue - let pulled_job = job; - if pulled_job.runnable_path.is_none() - || !has_concurent_limit - || pulled_job.canceled_by.is_some() - { - #[cfg(feature = "prometheus")] - if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { - QUEUE_PULL_COUNT.inc(); - } - otel_incr_queue_pull_count(); - return Ok(Some(PulledJobResult { - job: Some(pulled_job), - suspended, - missing_concurrency_key: false, - error_while_preprocessing: None, - })); - } - - #[cfg(feature = "private")] - if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING) { - return Ok(timeout( - Duration::from_secs(15), - crate::jobs_ee::apply_concurrency_limit( - db, - pull_loop_count, - suspended, - pulled_job, - &concurrency_settings, - ), + let concurrency_settings = windmill_common::runnable_settings::prefetch_cached_from_handle( + job.runnable_settings_handle, + db, ) - .await??); - } - #[cfg(not(feature = "private"))] - let _ = pull_loop_count; - Ok(None) -} + .await? + .1 + .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); -/// Claims jobs for several waiting workers in one statement per queue walked, instead of one -/// pull per worker. Each distinct name in `worker_names` gets at most one job, marked running -/// under that name; a worker missing from the result found nothing to run. -/// -/// Walks the queues the way [`pull`] does for one worker: when `suspend_first`, suspended flows -/// ready to resume on any of the tags; then `tag_groups` from highest priority down, with the -/// workspace-fairness share of the workers first restricted to uncapped workspaces. A job over -/// its concurrency limit is re-queued for later and its worker is pulled for again. -/// -/// No authorization happens here: jobs from every workspace matching the tags are claimed and -/// returned with their args and permissions. Callers must authenticate each worker named and -/// pass only tags that worker is allowed to serve. -pub async fn pull_batch( - db: &Pool, - tag_groups: &[Vec], - worker_names: &[String], - suspend_first: bool, -) -> windmill_common::error::Result> { - let mut waiting: Vec = worker_names.iter().unique().cloned().collect(); - let mut admitted = vec![]; - for pull_loop_count in 1..=PULL_LOOP_LIMIT { - if waiting.is_empty() { - break; + let has_concurent_limit = + has_active_concurrency_limit(concurrency_settings.concurrent_limit); + + #[cfg(not(feature = "enterprise"))] + if has_concurent_limit && !job.is_dependency() { + tracing::error!("Concurrent limits are an EE feature only, ignoring constraints") } - let claimed = match claim_batch(db, tag_groups, &waiting, suspend_first).await + + #[cfg(not(feature = "enterprise"))] + let has_concurent_limit = job.is_dependency() + && has_active_concurrency_limit(job.concurrent_limit) + && cfg!(feature = "private") + && !*WMDEBUG_NO_DEBOUNCING; + // if we don't have private flag, we don't have concurrency limit + + // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue + let pulled_job = job; + if pulled_job.runnable_path.is_none() + || !has_concurent_limit + || pulled_job.canceled_by.is_some() { - Ok(claimed) => claimed, - Err(e) if admitted.is_empty() => return Err(e), - // Jobs admitted by an earlier pass are running under their workers: return them. - Err(e) => { - tracing::error!( - "batch pull re-pull failed, returning the jobs admitted so far: {e:#}" - ); - break; - } - }; - let mut requeued = false; - for (worker_name, job, suspended) in claimed { - waiting.retain(|w| w != &worker_name); - let job_id = job.id; - match admit_pulled_job(db, job, suspended, pull_loop_count).await { - Ok(Some(res)) => admitted.push((worker_name, res)), - Ok(None) => { - waiting.push(worker_name); - requeued = true; - } - // The other claimed jobs are already running under their workers, so an error - // here must not abort the batch: hand this one back to the queue instead. - Err(e) => { - tracing::error!( - "error admitting batch-pulled job {job_id}, re-queuing it: {e:#}" - ); - if let Err(e) = sqlx::query( - "WITH ping AS ( - UPDATE v2_job_runtime SET ping = null WHERE id = $1 - ) - UPDATE v2_job_queue SET running = false, started_at = null - WHERE id = $1 AND worker = $2 AND running = true", - ) - .bind(job_id) - .bind(&worker_name) - .execute(db) - .await - { - tracing::error!("could not re-queue batch-pulled job {job_id}: {e:#}"); - } - } - } - } - if !requeued { - break; - } - if pull_loop_count == PULL_LOOP_LIMIT { - tracing::warn!( - "Batch pull loop count reached {PULL_LOOP_LIMIT}, backing off (likely concurrency re-queue storm)" - ); - } - } - Ok(admitted) -} - -/// One pass over the queues for `worker_names`, returning `(worker, job, suspended)` for each -/// job claimed. A failing statement ends the pass but keeps what was already claimed, since -/// those jobs are marked running and would otherwise be stranded until the zombie monitor. -async fn claim_batch( - db: &Pool, - tag_groups: &[Vec], - worker_names: &[String], - suspend_first: bool, -) -> windmill_common::error::Result> { - let mut claimed = vec![]; - let mut waiting = worker_names.to_vec(); - let pass: windmill_common::error::Result<()> = async { - let tags: Vec = tag_groups.iter().flatten().unique().cloned().collect(); - if suspend_first && !tags.is_empty() { - for (worker, job) in - claim_from_queue(db, &tags, PullQueue::Suspended, &[], &waiting).await? - { - waiting.retain(|w| w != &worker); - claimed.push((worker, job, true)); + #[cfg(feature = "prometheus")] + if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + QUEUE_PULL_COUNT.inc(); } + otel_incr_queue_pull_count(); + return Ok(PulledJobResult { + job: Some(pulled_job), + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }); } - crate::workspace_fairness::maybe_refresh_overloaded(db); - let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full(); - if !overloaded.is_empty() { - // Same admission draw `pull` makes once per worker, made here once per waiting worker. - let mut fair: Vec = waiting - .iter() - .filter(|_| !crate::workspace_fairness::should_admit_capped()) - .cloned() - .collect(); - for tags in tag_groups.iter().filter(|t| !t.is_empty()) { - if fair.is_empty() { - break; - } - for (worker, job) in claim_from_queue( + #[cfg(feature = "private")] + if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING) { + if let Some(pulled_job_res) = timeout( + Duration::from_secs(15), + crate::jobs_ee::apply_concurrency_limit( db, - tags, - PullQueue::ReadyFairness, - overloaded.as_slice(), - &fair, - ) - .await? - { - fair.retain(|w| w != &worker); - waiting.retain(|w| w != &worker); - claimed.push((worker, job, false)); - } - } - } - - for tags in tag_groups.iter().filter(|t| !t.is_empty()) { - if waiting.is_empty() { - break; - } - for (worker, job) in claim_from_queue(db, tags, PullQueue::Ready, &[], &waiting).await? + pull_loop_count, + suspended, + pulled_job, + &concurrency_settings, + ), + ) + .await?? { - waiting.retain(|w| w != &worker); - claimed.push((worker, job, false)); + return Ok(pulled_job_res); } } - Ok(()) } - .await; - match pass { - Err(e) if claimed.is_empty() => Err(e), - Err(e) => { - tracing::error!( - "batch pull pass failed after claiming {} jobs: {e:#}", - claimed.len() - ); - Ok(claimed) - } - Ok(()) => Ok(claimed), - } -} - -async fn claim_from_queue( - db: &Pool, - tags: &[String], - queue: PullQueue, - overloaded: &[String], - worker_names: &[String], -) -> windmill_common::error::Result> { - use sqlx::{FromRow, Row}; - let query = make_batch_pull_query(tags, queue); - let mut query = sqlx::query(&query).bind(worker_names); - if queue == PullQueue::ReadyFairness { - query = query.bind(overloaded); - } - let rows = timeout(Duration::from_secs(15), query.fetch_all(db)).await??; - rows.iter() - .map(|row| Ok((row.try_get("assigned_worker")?, PulledJob::from_row(row)?))) - .collect() } async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( From 054109d8559cc0a41ef361786c4d1d99eab9c502 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 29 Sep 2026 15:26:41 +0200 Subject: [PATCH 75/91] fix: create workspace forks in the background, with progress (#11406) * fix: create a fork in the background so a proxy timeout cannot cut it create_fork copies the whole workspace inside the request, which can run past the route timeout of an ingress in front of Windmill (Envoy's 15s default), and the UI then reports a failure for a fork still being made. create_fork?background=true now returns once the request is validated and records the copy in workspace_fork_creation, which the new fork_creation_status endpoint reads. The wizard and AI-session forks use it. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: drain background forks on shutdown and fork in the background from the CLI Co-Authored-By: Claude Opus 5.5 (1M context) * fix: settle fork polls by the fork's existence, adopt in-flight creations Co-Authored-By: Claude Opus 5.5 (1M context) * feat: show which part of the copy a fork being created is in The background copy reports its phase (data tables, settings, resources, scripts, flows, apps, drafts, triggers) through a watch channel. The heartbeat task records it on the fork's creation row as soon as it changes, and fork_creation_status returns it. The fork wizard shows it on its button, and the CLI logs each step. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: join a retried fork creation server-side, settle status by the fork's existence A retry from the same user and parent joins the creation in flight instead of being refused, so clients no longer match the refusal's wording, and another requester can never adopt it. The status route reports a fork that exists under its parent as completed, whatever its run's record says. Clients give up on failing polls after a time window rather than a count, which a rolling deploy can exhaust. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: drop fork-creation joins and existence probes A second request for a fork being created is refused again; only the AI-session fork, whose request never varies, waits for its own earlier one. Clients recognise a server without background forks by its synchronous answer instead of probing for a workspace by id, which could name another parent's fork. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: poll a background fork by the id of its own attempt create_fork?background=true answers with a creation id, and the status route reads that attempt only, for the user who started it. A retry that reuses the fork id is a new attempt, so a poller never reads another attempt's outcome. The AI-session fork no longer adopts a creation in flight, which it could not tell apart from someone else's. Co-Authored-By: Claude Opus 5.5 (1M context) * fix: record a fork's completion in its own commit, resume a session's fork after reload The attempt is marked complete in the transaction that creates the fork, so the status never infers completion from a workspace that may belong to another request. An AI session keeps the creation id on its pending fork and, after a reload mid-copy, waits for that attempt instead of requesting the fork again. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/Cargo.lock | 1 + backend/Cargo.toml | 2 +- ...929092707_workspace_fork_creation.down.sql | 1 + ...60929092707_workspace_fork_creation.up.sql | 20 + .../tests/fork_background.rs | 120 ++++++ backend/windmill-api-workspaces/Cargo.toml | 1 + .../windmill-api-workspaces/src/workspaces.rs | 346 +++++++++++++++--- backend/windmill-api/openapi.yaml | 43 ++- backend/windmill-api/src/lib.rs | 9 + cli/src/commands/workspace/fork.ts | 46 ++- .../sessions/sessionState.svelte.ts | 33 +- .../components/sessions/sessionState.test.ts | 33 ++ .../CreateWorkspaceInner.svelte | 23 +- frontend/src/lib/utils/forkCreation.test.ts | 63 ++++ frontend/src/lib/utils/forkCreation.ts | 57 +++ 15 files changed, 730 insertions(+), 68 deletions(-) create mode 100644 backend/migrations/20260929092707_workspace_fork_creation.down.sql create mode 100644 backend/migrations/20260929092707_workspace_fork_creation.up.sql create mode 100644 backend/windmill-api-integration-tests/tests/fork_background.rs create mode 100644 frontend/src/lib/utils/forkCreation.test.ts create mode 100644 frontend/src/lib/utils/forkCreation.ts diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e80d0edf08..4d73a953b8 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15551,6 +15551,7 @@ dependencies = [ "strum", "tokio", "tokio-postgres", + "tokio-util", "tracing", "url", "uuid", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 8571b0b99c..27d8aeac8e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -519,7 +519,7 @@ serde_urlencoded = "^0" astral-tokio-tar = "^0.6.2" tempfile = "^3" x509-parser = "^0.16" -tokio-util = { version = "=0.7.17", features = ["io"] } +tokio-util = { version = "=0.7.17", features = ["io", "rt"] } json-pointer = "^0" itertools = "^0.14.0" regex = "^1" diff --git a/backend/migrations/20260929092707_workspace_fork_creation.down.sql b/backend/migrations/20260929092707_workspace_fork_creation.down.sql new file mode 100644 index 0000000000..924855e129 --- /dev/null +++ b/backend/migrations/20260929092707_workspace_fork_creation.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS workspace_fork_creation; diff --git a/backend/migrations/20260929092707_workspace_fork_creation.up.sql b/backend/migrations/20260929092707_workspace_fork_creation.up.sql new file mode 100644 index 0000000000..8e3a678419 --- /dev/null +++ b/backend/migrations/20260929092707_workspace_fork_creation.up.sql @@ -0,0 +1,20 @@ +-- A fork being created in the background, and how that ended. The fork's own workspace row only +-- exists once the copy commits, so this is where a failure is reported to whoever polls for it. +-- `heartbeat_at` is refreshed while the copy runs: an unfinished row that stops being refreshed +-- belongs to a server that went away mid-copy. `step` names the part of the copy it is in. +-- `creation_id` names one attempt: the row is reused when a failed or abandoned one is retried, and +-- whoever polls reads only the attempt it started. +CREATE TABLE workspace_fork_creation ( + fork_workspace_id VARCHAR(50) PRIMARY KEY, + parent_workspace_id VARCHAR(50) NOT NULL, + creation_id UUID NOT NULL, + created_by VARCHAR(255) NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + step TEXT, + finished_at TIMESTAMPTZ, + error TEXT +); + +GRANT ALL ON workspace_fork_creation TO windmill_user; +GRANT ALL ON workspace_fork_creation TO windmill_admin; diff --git a/backend/windmill-api-integration-tests/tests/fork_background.rs b/backend/windmill-api-integration-tests/tests/fork_background.rs new file mode 100644 index 0000000000..0e82fb0d70 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/fork_background.rs @@ -0,0 +1,120 @@ +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +async fn wait_for_fork(client: &reqwest::Client, base_url: &str, creation_id: &str) -> Value { + for _ in 0..100 { + let status: Value = client + .get(format!("{base_url}/fork_creation_status/{creation_id}")) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + if status["status"] != "running" { + return status; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + panic!("fork creation {creation_id} still running"); +} + +/// A fork created in the background is reported by `fork_creation_status`, including a failure +/// that happens after the request has already returned. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_fork_created_in_background(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let base_url = format!( + "http://localhost:{}/api/w/test-workspace/workspaces", + server.addr.port() + ); + let client = reqwest::Client::new(); + let create = |id: &str| { + client + .post(format!("{base_url}/create_fork?background=true")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "id": id, "name": id })) + .send() + }; + + let resp = create("wm-fork-bg-ok").await?; + assert_eq!(resp.status(), 200); + let ok_creation = resp.text().await?; + let status = wait_for_fork(&client, &base_url, &ok_creation).await; + assert_eq!(status, json!({ "status": "completed" })); + // Nobody but the user who started a creation reads it. + let resp = client + .get(format!("{base_url}/fork_creation_status/{ok_creation}")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .send() + .await?; + assert_eq!(resp.status(), 404); + let exists: bool = + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM workspace WHERE id = 'wm-fork-bg-ok')") + .fetch_one(&db) + .await?; + assert!(exists); + // Under CE's workspace cap. + sqlx::query("UPDATE workspace SET deleted = true WHERE id = 'wm-fork-bg-ok'") + .execute(&db) + .await?; + + // Hold the fork's write lock so the request is validated and returns, then commit a workspace + // with the same id: the background write fails on it. + let mut blocker = db.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtext('fork:wm-fork-bg-err'))") + .execute(&mut *blocker) + .await?; + let resp = create("wm-fork-bg-err").await?; + assert_eq!(resp.status(), 200); + let err_creation = resp.text().await?; + let resp = create("wm-fork-bg-err").await?; + assert_eq!(resp.status(), 400, "a second creation while the first is in flight"); + sqlx::query("INSERT INTO workspace (id, name, owner) VALUES ('wm-fork-bg-err', 'x', 'x')") + .execute(&mut *blocker) + .await?; + blocker.commit().await?; + let status = wait_for_fork(&client, &base_url, &err_creation).await; + assert_eq!(status["status"], "failed"); + assert!( + status["error"].as_str().unwrap().contains("workspace_pkey"), + "{status}" + ); + + // A retry of the same id is a new attempt: the failed one does not read its outcome. + sqlx::query("DELETE FROM workspace WHERE id = 'wm-fork-bg-err'") + .execute(&db) + .await?; + let resp = create("wm-fork-bg-err").await?; + assert_eq!(resp.status(), 200); + let retry_creation = resp.text().await?; + assert_eq!( + wait_for_fork(&client, &base_url, &retry_creation).await, + json!({ "status": "completed" }) + ); + let resp = client + .get(format!("{base_url}/fork_creation_status/{err_creation}")) + .header("Authorization", "Bearer SECRET_TOKEN") + .send() + .await?; + assert_eq!(resp.status(), 404); + + // Without the flag the fork is created before the answer, which clients asking for a + // background fork recognise a server without it by. + sqlx::query("UPDATE workspace SET deleted = true WHERE id = 'wm-fork-bg-err'") + .execute(&db) + .await?; + let resp = client + .post(format!("{base_url}/create_fork")) + .header("Authorization", "Bearer SECRET_TOKEN") + .json(&json!({ "id": "wm-fork-sync", "name": "wm-fork-sync" })) + .send() + .await?; + assert_eq!(resp.text().await?, "Created forked workspace wm-fork-sync"); + + Ok(()) +} diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 7ca1f692ad..9f05a26f28 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -50,6 +50,7 @@ sha2.workspace = true sqlx.workspace = true tokio.workspace = true tokio-postgres.workspace = true +tokio-util.workspace = true tracing.workspace = true url.workspace = true uuid.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 458e63a973..31d049d3c0 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -176,6 +176,10 @@ pub fn workspaced_service() -> Router { .route("/leave", post(leave_workspace)) .route("/get_workspace_name", get(get_workspace_name)) .route("/create_fork", post(create_workspace_fork)) + .route( + "/fork_creation_status/{creation_id}", + get(get_fork_creation_status), + ) .route("/attach_dev_workspace", post(attach_dev_workspace)) .route("/detach_dev_workspace", post(detach_dev_workspace)) .route("/get_dev_workspace", get(get_dev_workspace)) @@ -6924,7 +6928,9 @@ async fn clone_workspace_data( source_workspace_id: &str, target_workspace_id: &str, authed: &ApiAuthed, + progress: &ForkProgress, ) -> Result<()> { + progress.step("Copying settings, folders and groups"); // Clone workspace settings (merge with existing basic settings) update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; @@ -6946,6 +6952,7 @@ async fn clone_workspace_data( // Clone groups clone_groups(tx, source_workspace_id, target_workspace_id).await?; + progress.step("Copying resources and variables"); // Clone resource types clone_resource_types(tx, source_workspace_id, target_workspace_id).await?; @@ -6955,6 +6962,7 @@ async fn clone_workspace_data( // Clone variables (including external secret backend replication) clone_variables(tx, db, source_workspace_id, target_workspace_id).await?; + progress.step("Copying scripts"); // Clone scripts with new hashes clone_scripts(tx, source_workspace_id, target_workspace_id).await?; @@ -6972,18 +6980,21 @@ async fn clone_workspace_data( clone_metric_catalog(tx, source_workspace_id, target_workspace_id).await?; clone_asset_usages_and_triggers(tx, source_workspace_id, target_workspace_id).await?; + progress.step("Copying flows"); // Clone flows with new versions clone_flows(tx, source_workspace_id, target_workspace_id).await?; // Clone flow nodes clone_flow_nodes(tx, source_workspace_id, target_workspace_id).await?; + progress.step("Copying apps"); // Clone apps with new IDs and app scripts let _app_id_mapping = clone_apps(tx, source_workspace_id, target_workspace_id, authed).await?; // Clone raw apps clone_raw_apps(tx, source_workspace_id, target_workspace_id).await?; + progress.step("Copying drafts and dependencies"); // Clone the forker's own per-user drafts (plus the legacy NULL-email // workspace draft, if any) so they keep their pending edits in the // fork. Other users' drafts are intentionally NOT cloned — they don't @@ -9209,6 +9220,7 @@ async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, Path(parent_workspace_id): Path, + Query(q): Query, Json(nw): Json, ) -> Result { enforce_fork_depth(&db, &parent_workspace_id, 0).await?; @@ -9314,63 +9326,279 @@ async fn create_workspace_fork( // Refused here, before any database exists; `make_copies` checks again under the locks. validate_forked_datatables(&db, &authed, &parent_workspace_id, &nw).await?; - // Detached from the request: a client that goes away while the copies are made must still end - // with the fork created or no copy left behind, and dropping the handler's future would skip - // that cleanup. - tokio::spawn(async move { - let fork_id = nw.id.clone(); - let copies = crate::datatable_clone::make_copies( - &db, - &authed, - &parent_workspace_id, - ©_requests(&nw.forked_datatables), - ) - .await?; - let replayed: Vec = copies - .iter() - .filter(|c| c.replayed) - .map(|c| c.behavior) - .collect(); - match write_workspace_fork( + if !q.background.unwrap_or(false) { + // Detached from the request: a client that goes away while the copies are made must still + // end with the fork created or no copy left behind, and dropping the handler's future + // would skip that cleanup. + return tokio::spawn(make_workspace_fork( + db, + authed, + parent_workspace_id, + nw, + dev_workspace_label, + ForkProgress::default(), + )) + .await + .map_err(|e| { + Error::internal_err(format!("Creating the fork stopped unexpectedly: {e}")) + })?; + } + + let fork_id = nw.id.clone(); + sqlx::query( + "DELETE FROM workspace_fork_creation + WHERE COALESCE(finished_at, heartbeat_at) < now() - interval '7 days'", + ) + .execute(&db) + .await?; + let claimed = sqlx::query( + "INSERT INTO workspace_fork_creation + (fork_workspace_id, parent_workspace_id, created_by, creation_id) + VALUES ($1, $2, $3, $5) + ON CONFLICT (fork_workspace_id) DO UPDATE SET + parent_workspace_id = EXCLUDED.parent_workspace_id, + created_by = EXCLUDED.created_by, + creation_id = EXCLUDED.creation_id, + started_at = now(), heartbeat_at = now(), step = NULL, finished_at = NULL, + error = NULL + WHERE workspace_fork_creation.finished_at IS NOT NULL + OR workspace_fork_creation.heartbeat_at < now() - $4 * interval '1 second' + RETURNING creation_id", + ) + .bind(&fork_id) + .bind(&parent_workspace_id) + .bind(&authed.email) + .bind(FORK_HEARTBEAT_STALE_SECS as f64) + .bind(Uuid::new_v4()) + .fetch_optional(&db) + .await?; + // Every write of this attempt is scoped to its `creation_id`: one taken over as abandoned must + // not report over the attempt that took its place. + let Some(creation_id) = claimed.map(|r| r.get::("creation_id")) else { + return Err(Error::BadRequest(format!( + "workspace '{fork_id}' is already being created" + ))); + }; + + BACKGROUND_FORKS.spawn(async move { + let (progress_tx, mut progress_rx) = tokio::sync::watch::channel(""); + // The one writer of the run's row while it copies: it records a new step as soon as the + // copy reports it, and refreshes the heartbeat in between. + let heartbeat = { + let (db, fork_id) = (db.clone(), fork_id.clone()); + tokio::spawn(async move { + let mut interval = + tokio::time::interval(std::time::Duration::from_secs(FORK_HEARTBEAT_SECS)); + let mut reporting = true; + loop { + tokio::select! { + _ = interval.tick() => {} + changed = progress_rx.changed(), if reporting => reporting = changed.is_ok(), + } + let step = *progress_rx.borrow_and_update(); + if let Err(e) = sqlx::query( + "UPDATE workspace_fork_creation SET heartbeat_at = now(), step = $3 + WHERE fork_workspace_id = $1 AND creation_id = $2", + ) + .bind(&fork_id) + .bind(creation_id) + .bind((!step.is_empty()).then_some(step)) + .execute(&db) + .await + { + tracing::warn!("fork '{fork_id}': could not record progress: {e}"); + } + } + }) + }; + let made = tokio::spawn(make_workspace_fork( db.clone(), authed, parent_workspace_id, nw, dev_workspace_label, - &copies, + ForkProgress { creation_id: Some(creation_id), steps: Some(progress_tx) }, + )) + .await + .map_err(|e| Error::internal_err(format!("Creating the fork stopped unexpectedly: {e}"))) + .and_then(|r| r); + heartbeat.abort(); + let error = made.err().map(|e| { + tracing::error!("Creating fork '{fork_id}' failed: {e}"); + e.to_string() + }); + // A fork that committed is already recorded complete, even when its commit's + // acknowledgement was lost and the attempt reads as an error here. + if let Err(e) = sqlx::query( + "UPDATE workspace_fork_creation SET finished_at = now(), error = $3 + WHERE fork_workspace_id = $1 AND creation_id = $2 AND finished_at IS NULL", ) + .bind(&fork_id) + .bind(creation_id) + .bind(error) + .execute(&db) .await { - Ok(message) => { - for behavior in replayed { - windmill_common::feature_usage::log_feature_usage( - "datatable", - "clone_replayed", - match behavior { - DataTableForkBehavior::SchemaOnly => "schema_only", - _ => "schema_and_data", - }, - ); - } - Ok(message) - } - // A commit whose acknowledgement was lost can still have committed, and a concurrent - // request for the same id can have: once the write has settled, the cleanup keeps - // whichever copies a committed fork names, and drops the rest. - Err(e) => match wait_for_fork_write(&db, &fork_id).await { - Ok(()) => Err(crate::datatable_clone::drop_copies_after(&db, copies, e).await), - Err(settle) => { - tracing::error!( - "Could not tell whether fork '{fork_id}' was created, so its copies were \ - kept: {settle}" - ); - Err(e) - } - }, + tracing::error!("fork '{fork_id}': could not record how its creation ended: {e}"); } - }) + }); + Ok(creation_id.to_string()) +} + +/// How often a fork created in the background records that its copy is still running, and how long +/// without that record its creation counts as abandoned. +const FORK_HEARTBEAT_SECS: u64 = 10; +const FORK_HEARTBEAT_STALE_SECS: u64 = 60; + +/// Forks being created in the background. The server waits for them on shutdown: a copy cut short +/// never drops the data table databases it already made, and a retry under the same id is refused +/// while they exist. +pub static BACKGROUND_FORKS: std::sync::LazyLock = + std::sync::LazyLock::new(tokio_util::task::TaskTracker::new); + +#[derive(Deserialize)] +struct CreateWorkspaceForkQuery { + /// Return once the request is validated and create the fork in the background, answering with + /// the id `fork_creation_status` reads this attempt by. Without it the answer is + /// `Created forked workspace `, which clients take as a server without background forks. + background: Option, +} + +/// The attempt a fork created in the background belongs to, and where its copy is, for its status +/// to report. +#[derive(Default)] +struct ForkProgress { + creation_id: Option, + steps: Option>, +} + +impl ForkProgress { + fn step(&self, step: &'static str) { + if let Some(tx) = &self.steps { + tx.send_replace(step); + } + } +} + +#[derive(Serialize)] +struct ForkCreationStatus { + /// `running`, `completed` or `failed`. + status: &'static str, + /// The part of the copy a running fork is in. + #[serde(skip_serializing_if = "Option::is_none")] + step: Option, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +async fn get_fork_creation_status( + authed: ApiAuthed, + Extension(db): Extension, + Path((parent_workspace_id, creation_id)): Path<(String, Uuid)>, +) -> JsonResult { + let row = sqlx::query( + "SELECT c.created_by, c.finished_at IS NOT NULL AS finished, c.error, c.step, + c.heartbeat_at < now() - $3 * interval '1 second' AS stale + FROM workspace_fork_creation c + WHERE c.creation_id = $1 AND c.parent_workspace_id = $2", + ) + .bind(creation_id) + .bind(&parent_workspace_id) + .bind(FORK_HEARTBEAT_STALE_SECS as f64) + .fetch_optional(&db) + .await?; + // Only the user who started it reads an attempt: its id is not handed to anyone else. + let Some(row) = row.filter(|r| r.get::("created_by") == authed.email) else { + return Err(Error::NotFound(format!( + "no fork creation {creation_id} from '{parent_workspace_id}'" + ))); + }; + let error: Option = row.get("error"); + let status = if row.get::("finished") { + match error { + Some(error) => ForkCreationStatus { status: "failed", step: None, error: Some(error) }, + None => ForkCreationStatus { status: "completed", step: None, error: None }, + } + } else if row.get::("stale") { + ForkCreationStatus { + status: "failed", + step: None, + error: Some( + "the server creating the fork stopped before it finished; check the server logs \ + and whether the fork exists before retrying" + .to_string(), + ), + } + } else { + ForkCreationStatus { status: "running", step: row.get("step"), error: None } + }; + Ok(Json(status)) +} + +/// Copy the data tables the fork gets, then write the fork itself, dropping the copies again if +/// that write fails. +async fn make_workspace_fork( + db: DB, + authed: ApiAuthed, + parent_workspace_id: String, + nw: CreateWorkspaceFork, + dev_workspace_label: Option, + progress: ForkProgress, +) -> Result { + let fork_id = nw.id.clone(); + if !nw.forked_datatables.is_empty() { + progress.step("Copying data tables"); + } + let copies = crate::datatable_clone::make_copies( + &db, + &authed, + &parent_workspace_id, + ©_requests(&nw.forked_datatables), + ) + .await?; + let replayed: Vec = copies + .iter() + .filter(|c| c.replayed) + .map(|c| c.behavior) + .collect(); + match write_workspace_fork( + db.clone(), + authed, + parent_workspace_id, + nw, + dev_workspace_label, + &copies, + &progress, + ) .await - .map_err(|e| Error::internal_err(format!("Creating the fork stopped unexpectedly: {e}")))? + { + Ok(message) => { + for behavior in replayed { + windmill_common::feature_usage::log_feature_usage( + "datatable", + "clone_replayed", + match behavior { + DataTableForkBehavior::SchemaOnly => "schema_only", + _ => "schema_and_data", + }, + ); + } + Ok(message) + } + // A commit whose acknowledgement was lost can still have committed, and a concurrent + // request for the same id can have: once the write has settled, the cleanup keeps + // whichever copies a committed fork names, and drops the rest. + Err(e) => match wait_for_fork_write(&db, &fork_id).await { + Ok(()) => Err(crate::datatable_clone::drop_copies_after(&db, copies, e).await), + Err(settle) => { + tracing::error!( + "Could not tell whether fork '{fork_id}' was created, so its copies were \ + kept: {settle}" + ); + Err(e) + } + }, + } } /// The database a data table of the fork or dev workspace `fork_id` is copied into, as the wizard @@ -9466,6 +9694,7 @@ async fn write_workspace_fork( nw: CreateWorkspaceFork, dev_workspace_label: Option, copies: &[crate::datatable_clone::MadeCopy], + progress: &ForkProgress, ) -> Result { let mut tx: Transaction<'_, Postgres> = db.begin().await?; // Held until this transaction ends: after an error, the copies' cleanup waits on it, so it reads @@ -9568,8 +9797,15 @@ async fn write_workspace_fork( .await?; // Clone all data from the parent workspace using Rust implementation - if let Err(e) = - clone_workspace_data(&mut tx, &db, &parent_workspace_id, &forked_id, &authed).await + if let Err(e) = clone_workspace_data( + &mut tx, + &db, + &parent_workspace_id, + &forked_id, + &authed, + progress, + ) + .await { // A genuine `\u0000` in a source `json` value (`app_version.value` / // `flow_version.schema`) aborts the clone when it is re-encoded to jsonb: @@ -9600,6 +9836,7 @@ async fn write_workspace_fork( // enabled=false. Disabled rows have no side effects (no listener // attaches, no cron fires) so this is safe by construction. The user // re-enables in the fork, with parent-conflict warnings on enable. + progress.step("Copying triggers and schedules"); clone_triggers_and_schedules(&mut tx, &parent_workspace_id, &forked_id).await?; repoint_unresolvable_cloned_identities(&mut tx, &forked_id, &authed).await?; @@ -9747,6 +9984,16 @@ async fn write_workspace_fork( (!copied.is_empty()).then(|| [("copied_datatables", copied.as_str())].into()), ) .await?; + // Committed with the fork, so its attempt reads complete exactly when the fork exists. + if let Some(creation_id) = progress.creation_id { + sqlx::query( + "UPDATE workspace_fork_creation SET finished_at = now(), error = NULL + WHERE creation_id = $1", + ) + .bind(creation_id) + .execute(&mut *tx) + .await?; + } tx.commit().await?; // A pre-creation lookup could have cached an EMPTY ancestor chain for this id, which @@ -9760,6 +10007,7 @@ async fn write_workspace_fork( windmill_common::workspaces::invalidate_protection_rules_cache(&parent_workspace_id); } + // Clients that ask for a background fork recognise a server without it by this answer. Ok(format!("Created forked workspace {}", &forked_id)) } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index cf373241d4..e7afd78ceb 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1398,6 +1398,14 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: background + description: | + return once the request is validated and create the fork in the background, answering + with the creation id getForkCreationStatus reads + in: query + required: false + schema: + type: boolean requestBody: description: new forked workspace required: true @@ -1407,12 +1415,45 @@ paths: $ref: "#/components/schemas/CreateWorkspaceFork" responses: "201": - description: forked workspace created + description: forked workspace created, or with background set, the id of its creation content: text/plain: schema: type: string + /w/{workspace}/workspaces/fork_creation_status/{creation_id}: + get: + summary: get the status of a fork created in the background + operationId: getForkCreationStatus + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: creation_id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: fork creation status + content: + application/json: + schema: + type: object + properties: + status: + type: string + enum: [running, completed, failed] + step: + type: string + description: the part of the copy a running fork is in + error: + type: string + required: + - status + /w/{workspace}/workspaces/attach_dev_workspace: post: summary: attach an existing workspace as the dev workspace of this (prod) workspace diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 156c6ca898..556163ce1c 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -1279,6 +1279,15 @@ pub async fn run_server( }); server.await?; + let forks = &windmill_api_workspaces::workspaces::BACKGROUND_FORKS; + forks.close(); + if !forks.is_empty() { + tracing::info!( + "server off. waiting for {} fork(s) being created", + forks.len() + ); + forks.wait().await; + } #[cfg(feature = "agent_worker_server")] for (i, bg_processor) in agent_workers_bg_processor.into_iter().enumerate() { tracing::info!("server off. shutting down agent worker bg processor {i}"); diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 0aca0984ff..eedd9c0842 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -13,6 +13,7 @@ import { renameCurrentGitBranch, } from "../../utils/git.ts"; import process from "node:process"; +import { apiErrorMessage } from "../../utils/utils.ts"; import { WM_FORK_PREFIX } from "../../core/constants.ts"; import { tryResolveBranchWorkspace } from "../../core/context.ts"; import { @@ -22,6 +23,38 @@ import { readConfigFile, } from "../../core/conf.ts"; +/** Wait for a fork creation started in the background, by the id its request answered with. */ +async function waitForForkCreation(parentWorkspace: string, creationId: string) { + let failingSince: number | undefined; + let lastStep: string | undefined; + while (true) { + let result: + | Awaited> + | undefined; + try { + result = await wmill.getForkCreationStatus({ + workspace: parentWorkspace, + creationId, + }); + failingSince = undefined; + } catch (e) { + // A failed poll says nothing about the fork: it can be lost to a proxy, or reach a replica + // that has no status route yet during a rolling deploy. + failingSince ??= Date.now(); + if (Date.now() - failingSince >= 3 * 60 * 1000) throw e; + } + if (result?.status === "running" && result.step && result.step !== lastStep) { + lastStep = result.step; + log.info(colors.gray(`${lastStep}...`)); + } + if (result?.status === "completed") return; + if (result?.status === "failed") { + throw new Error(result.error ?? "Unknown error"); + } + await new Promise((resolve) => setTimeout(resolve, 1500)); + } +} + async function createWorkspaceFork( opts: GlobalOptions & { createWorkspaceName: string | undefined; @@ -376,8 +409,9 @@ async function createWorkspaceFork( // --- Create the fork workspace --- try { - const result = await wmill.createWorkspaceFork({ + const response = await wmill.createWorkspaceFork({ workspace: workspace.workspaceId, + background: true, requestBody: { id: trueWorkspaceId, name: opts.createWorkspaceName ?? workspaceName ?? trueWorkspaceId, @@ -385,11 +419,17 @@ async function createWorkspaceFork( forked_datatables: forkedDatatables, }, }); + // A server without background forks ignores the flag and answers once the fork is created. + if (!response.startsWith("Created forked workspace")) { + await waitForForkCreation(workspace.workspaceId, response); + } - log.info(colors.green(`✅ ${result}`)); + log.info(colors.green(`✅ Created forked workspace ${trueWorkspaceId}`)); } catch (error) { log.error( - colors.red(`Failed to create forked workspace: ${(error as Error).message}`), + colors.red( + `Failed to create forked workspace: ${apiErrorMessage(error) ?? (error as Error).message}`, + ), ); throw error; } diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index c5a2f28455..e81df02120 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -45,6 +45,7 @@ export function syncWorkspaceTo(workspaceId: string | undefined): void { switchWorkspace(workspaceId) } import { WorkspaceService } from '$lib/gen' +import { createWorkspaceForkAndWait, waitForForkCreation } from '$lib/utils/forkCreation' import { sendUserToast } from '$lib/toast' import { onUserChange } from '$lib/userScopedStorage' @@ -76,6 +77,9 @@ export type PendingFork = { id: string // Display name shown in the workspace bar. name: string + // The background creation this fork started, so a reload mid-copy waits for it rather than + // requesting the fork again, which the server refuses while that creation runs. + creation_id?: string } export type SessionSummarySource = 'placeholder' | 'generated' | 'manual' @@ -1180,7 +1184,10 @@ export async function commitSessionWorkspace( `Community edition is limited to ${CE_MAX_NON_ADMIN_WORKSPACES + 1} workspaces — archive a workspace or pick one to run in` ) } - const newId = await materializeFork(fork) + const newId = await materializeFork(fork, (creation_id) => { + s.pending_fork = { ...fork, creation_id } + void putSession(s) + }) if (!newId) { // Real failure (not a recovered duplicate). Drop the pending // fork so the session falls through to the workspace-pick @@ -1297,13 +1304,27 @@ export function setGeneratedSessionSummary( // lost). Adopt it silently instead of re-POSTing — the API would // otherwise reject with workspace_pkey. Likewise, if the API returns a // duplicate-key error we refresh the store and adopt the existing row. -export async function materializeFork(fork: PendingFork): Promise { +export async function materializeFork( + fork: PendingFork, + onCreationStarted?: (creationId: string) => void +): Promise { if (get(userWorkspaces).some((w) => w.id === fork.id)) return fork.id try { - await WorkspaceService.createWorkspaceFork({ - workspace: fork.parent_workspace_id, - requestBody: { id: fork.id, name: fork.name } - }) + let resumed = false + if (fork.creation_id) { + // An attempt that failed or is gone is requested again below. + resumed = await waitForForkCreation(fork.parent_workspace_id, fork.creation_id).then( + () => true, + () => false + ) + } + if (!resumed) { + await createWorkspaceForkAndWait( + fork.parent_workspace_id, + { id: fork.id, name: fork.name }, + { onCreationStarted } + ) + } usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) sendUserToast(`Created fork ${fork.name}`) return fork.id diff --git a/frontend/src/lib/components/sessions/sessionState.test.ts b/frontend/src/lib/components/sessions/sessionState.test.ts index 5cf3e3269e..62d2b8d39f 100644 --- a/frontend/src/lib/components/sessions/sessionState.test.ts +++ b/frontend/src/lib/components/sessions/sessionState.test.ts @@ -37,6 +37,7 @@ vi.mock('$lib/gen', async (orig) => { WorkspaceService: { ...actual.WorkspaceService, createWorkspaceFork: vi.fn().mockRejectedValue(new Error('fork creation failed')), + getForkCreationStatus: vi.fn().mockResolvedValue({ status: 'completed' }), listUserWorkspaces: vi.fn().mockResolvedValue([]) } } @@ -108,6 +109,38 @@ describe('commitSessionWorkspace — fork-creation failure', () => { }) }) +describe('commitSessionWorkspace — fork creation resumed after a reload', () => { + it('waits for the creation the pending fork started instead of requesting it again', async () => { + const id = 'test-commit-fork-resume' + const prevLicense = get(enterpriseLicense) + enterpriseLicense.set('test-license') + vi.mocked(WorkspaceService.createWorkspaceFork).mockClear() + sessionState.sessions.push({ + id, + name: 'fork-resume', + createdAt: 0, + pending_fork: { + parent_workspace_id: 'parent_ws', + id: 'wm-fork-resume', + name: 'resume', + creation_id: '5c3e1b1e-0000-4000-8000-000000000000' + } + } as Session) + try { + expect(await commitSessionWorkspace(id, 'parent_ws')).toBe('wm-fork-resume') + expect(WorkspaceService.getForkCreationStatus).toHaveBeenCalledWith({ + workspace: 'parent_ws', + creationId: '5c3e1b1e-0000-4000-8000-000000000000' + }) + expect(WorkspaceService.createWorkspaceFork).not.toHaveBeenCalled() + } finally { + const i = sessionState.sessions.findIndex((x) => x.id === id) + if (i >= 0) sessionState.sessions.splice(i, 1) + enterpriseLicense.set(prevLicense) + } + }) +}) + // Set the workspace list to a given number of non-'admins' workspaces so the // commit-path's CE workspace-cap check (mirror of backend _check_nb_of_workspaces) // can be exercised. Returns a restore fn. diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte index d0f86b36ad..e6a0dce4fd 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte @@ -70,6 +70,7 @@ import WorkspaceScopeTrigger from '../WorkspaceScopeTrigger.svelte' import DarkModeToggle from '../sidebar/DarkModeToggle.svelte' import ForkDucklakeSection from './ForkDucklakeSection.svelte' + import { createWorkspaceForkAndWait } from '$lib/utils/forkCreation' interface Props { isFork?: boolean @@ -329,6 +330,7 @@ } let forkCreationLoading = $state(false) + let forkCreationStep = $state('Creating branch') let forkCreationError = $state('') let errorMsgs: string[] = $state([]) let failedSyncJobs: string[] = $state([]) @@ -376,6 +378,7 @@ async function forkWorkspace(prefixed_id: string): Promise { if (baseWorkspaceId) { forkCreationLoading = true + forkCreationStep = 'Creating branch' errorMsgs = [] failedSyncJobs = [] forkCreationError = '' @@ -470,9 +473,10 @@ } try { - await WorkspaceService.createWorkspaceFork({ - workspace: baseWorkspaceId!, - requestBody: { + forkCreationStep = 'Creating fork' + await createWorkspaceForkAndWait( + baseWorkspaceId!, + { id: prefixed_id, name, color: colorEnabled && workspaceColor ? workspaceColor : undefined, @@ -483,13 +487,15 @@ lock_prod_deploy: createAsDevWorkspace && effectiveLockProdDeploy, lock_prod_forking: createAsDevWorkspace && effectiveLockProdForking, copy_members: copyMembers - } - }) + }, + { onStep: (step) => (forkCreationStep = step) } + ) } catch (e) { + const msg = e?.body ?? e?.message ?? e ?? 'Unknown error' forkCreationError = `Failed to create fork '${prefixed_id}'` - errorMsgs.push(e?.body ?? e ?? 'Unknown error') + errorMsgs.push(msg) forkCreationLoading = false - sendUserToast(`Could not create fork '${prefixed_id}' ${e}`, true) + sendUserToast(`Could not create fork '${prefixed_id}' ${msg}`, true) return } @@ -1115,7 +1121,8 @@ {:else} {/if} diff --git a/frontend/src/lib/utils/forkCreation.test.ts b/frontend/src/lib/utils/forkCreation.test.ts new file mode 100644 index 0000000000..bf180c6c6f --- /dev/null +++ b/frontend/src/lib/utils/forkCreation.test.ts @@ -0,0 +1,63 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/gen', () => ({ + WorkspaceService: { + createWorkspaceFork: vi.fn(), + getForkCreationStatus: vi.fn() + } +})) + +import { WorkspaceService } from '$lib/gen' +import { createWorkspaceForkAndWait } from './forkCreation' + +const svc = vi.mocked(WorkspaceService) +const fork = { id: 'wm-fork-x', name: 'x' } + +async function run(onStep?: (step: string) => void) { + const p = createWorkspaceForkAndWait('ws', fork, { onStep }) + const settled = p.then( + () => 'completed', + (e) => `failed: ${e.message}` + ) + await vi.runAllTimersAsync() + return settled +} + +describe('createWorkspaceForkAndWait', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.resetAllMocks() + svc.createWorkspaceFork.mockResolvedValue('5c3e1b1e-0000-4000-8000-000000000000') + }) + afterEach(() => vi.useRealTimers()) + + it('reports each step while the fork runs, then how it ended', async () => { + svc.getForkCreationStatus + .mockResolvedValueOnce({ status: 'running' }) + .mockResolvedValueOnce({ status: 'running', step: 'Copying flows' }) + .mockResolvedValueOnce({ status: 'failed', error: 'boom' }) + const steps: string[] = [] + expect(await run((s) => steps.push(s))).toBe('failed: boom') + expect(steps).toEqual(['Copying flows']) + }) + + it('returns at once when the server created the fork synchronously', async () => { + // A server without background forks ignores the flag. + svc.createWorkspaceFork.mockResolvedValue('Created forked workspace wm-fork-x') + expect(await run()).toBe('completed') + expect(svc.getForkCreationStatus).not.toHaveBeenCalled() + }) + + it('keeps polling through failed polls', async () => { + svc.getForkCreationStatus + .mockRejectedValueOnce(Object.assign(new Error('Not Found'), { status: 404 })) + .mockRejectedValueOnce(new Error('upstream request timeout')) + .mockResolvedValueOnce({ status: 'completed' }) + expect(await run()).toBe('completed') + }) + + it('gives up once polls keep failing for a fork that never appears', async () => { + svc.getForkCreationStatus.mockRejectedValue(new Error('upstream request timeout')) + expect(await run()).toBe('failed: upstream request timeout') + }) +}) diff --git a/frontend/src/lib/utils/forkCreation.ts b/frontend/src/lib/utils/forkCreation.ts new file mode 100644 index 0000000000..d0fb4e5edf --- /dev/null +++ b/frontend/src/lib/utils/forkCreation.ts @@ -0,0 +1,57 @@ +import { WorkspaceService, type CreateWorkspaceFork } from '$lib/gen' + +const POLL_INTERVAL_MS = 1500 +// How long polls may keep failing, which a rolling deploy's replicas without the status route cause. +const MAX_FAILING_POLLS_MS = 3 * 60 * 1000 + +/** + * Create a fork and resolve once it exists, rejecting with the reason it could not be created. + * + * The copy runs on the server after the request returns: a large workspace takes longer to copy + * than the timeout of many proxies in front of Windmill, which would otherwise cut the request. + * `onCreationStarted` receives the id `waitForForkCreation` resumes the wait by, and `onStep` the + * part of the copy the server reports it is in. + */ +export async function createWorkspaceForkAndWait( + parentWorkspace: string, + fork: CreateWorkspaceFork, + opts: { onCreationStarted?: (creationId: string) => void; onStep?: (step: string) => void } = {} +): Promise { + const response = await WorkspaceService.createWorkspaceFork({ + workspace: parentWorkspace, + background: true, + requestBody: fork + }) + // A server without background forks ignores the flag and answers once the fork is created. + if (response.startsWith('Created forked workspace')) return + opts.onCreationStarted?.(response) + await waitForForkCreation(parentWorkspace, response, opts.onStep) +} + +/** Wait for a fork creation started in the background, by the id its request answered with. */ +export async function waitForForkCreation( + parentWorkspace: string, + creationId: string, + onStep?: (step: string) => void +): Promise { + let failingSince: number | undefined + while (true) { + let result: Awaited> | undefined + try { + result = await WorkspaceService.getForkCreationStatus({ + workspace: parentWorkspace, + creationId + }) + failingSince = undefined + } catch (e) { + // A failed poll says nothing about the fork: it can be lost to the same proxy the fork + // runs in the background to avoid, or reach a replica that has no status route yet. + failingSince ??= Date.now() + if (Date.now() - failingSince >= MAX_FAILING_POLLS_MS) throw e + } + if (result?.status === 'running' && result.step) onStep?.(result.step) + if (result?.status === 'completed') return + if (result?.status === 'failed') throw new Error(result.error ?? 'Unknown error') + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)) + } +} From da5c58de2a0c297cfa81a601f9f543f46107e3f6 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:39:10 +0200 Subject: [PATCH 76/91] fix: keep secret variable values hidden when toggling secret (#11383) * fix: keep secret variable values hidden when toggling secret off and on Co-Authored-By: Claude Opus 5.5 (1M context) * fix: clarify the secret unlock hint and hide it from read-only users Co-Authored-By: Claude Opus 5.5 (1M context) * fix: name the secret toggle and skip the load lock on draft-only variables Co-Authored-By: Claude Opus 5.5 (1M context) * fix: name the secret toggle with aria-label so its heading does not flip it Co-Authored-By: Claude Opus 5.5 (1M context) * fix: re-hide a cleared secret value when turning secret back on Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- frontend/src/lib/components/Toggle.svelte | 4 ++ .../src/lib/components/VariableEditor.svelte | 17 ++++++++ .../src/lib/components/VariableForm.svelte | 42 ++++++++++++++----- 3 files changed, 53 insertions(+), 10 deletions(-) diff --git a/frontend/src/lib/components/Toggle.svelte b/frontend/src/lib/components/Toggle.svelte index 200b449ee7..37a2ebbebe 100644 --- a/frontend/src/lib/components/Toggle.svelte +++ b/frontend/src/lib/components/Toggle.svelte @@ -32,6 +32,8 @@ class?: string | undefined size?: '2xs' | 'xs' | 'sm' | 'md' textDisabled?: boolean + /** Accessible name for a toggle whose visible label sits outside it. */ + ariaLabel?: string right?: import('svelte').Snippet } @@ -50,6 +52,7 @@ class: className = undefined, size = 'sm', textDisabled = false, + ariaLabel = undefined, right }: Props = $props() @@ -112,6 +115,7 @@ {disabled} type="checkbox" {id} + aria-label={ariaLabel} class="sr-only peer" bind:checked onchange={stopPropagation((e) => { diff --git a/frontend/src/lib/components/VariableEditor.svelte b/frontend/src/lib/components/VariableEditor.svelte index fe6ccd1f27..be76185959 100644 --- a/frontend/src/lib/components/VariableEditor.svelte +++ b/frontend/src/lib/components/VariableEditor.svelte @@ -409,6 +409,19 @@ form?.setCode(getV.value ?? '') } + // Re-securing hides the deployed value again, back to the state before it was loaded. + // A value the user typed is theirs to keep: it is what the next save deploys. + function onSecretChange(isSecret: boolean): void { + if (!isSecret || !selected || !existedInitially[selected]) return + const s = states[selected]?.draft + const ini = initialStates[selected] + if (!s || !ini?.variable.is_secret) return + if (s.variable.value !== '' && s.variable.value !== ini.variable.value) return + s.variable.value = '' + ini.variable.value = '' + form?.setCode('') + } + async function save(): Promise { const dirty = dirtyWorkspaces const savedPath = (curWs ? states[curWs]?.draft?.path : undefined) ?? editPath ?? '' @@ -549,6 +562,10 @@ can_write={can_write === true} {edit} onLoadSecret={loadSecret} + {onSecretChange} + deployedSecret={!!selected && + !!existedInitially[selected] && + !!initialStates[selected]?.variable.is_secret} workspace={selected} actingUser={acting.in(selected) ?? null} /> diff --git a/frontend/src/lib/components/VariableForm.svelte b/frontend/src/lib/components/VariableForm.svelte index 63fdbf7425..272004396c 100644 --- a/frontend/src/lib/components/VariableForm.svelte +++ b/frontend/src/lib/components/VariableForm.svelte @@ -35,6 +35,10 @@ can_write: boolean edit: boolean onLoadSecret?: () => void + /** Fires after the user flips the Secret toggle, with its new state. */ + onSecretChange?: (isSecret: boolean) => void + /** Whether the deployed variable being edited is secret. */ + deployedSecret?: boolean /** Workspace the path is validated against; defaults to the nav workspace. */ workspace?: string | undefined /** The user acting in `workspace`, resolved by the editor above. `undefined` while @@ -55,6 +59,8 @@ can_write, edit, onLoadSecret, + onSecretChange, + deployedSecret = false, workspace = undefined, actingUser }: Props = $props() @@ -67,6 +73,10 @@ // '' is the sentinel for "stages nothing", matching the deploy bodies. let hasStagedValue = $derived(variable.value !== '') + // Un-securing has to send the plaintext, and fetching it silently on the toggle would + // reveal the secret without the explicit, audited "Load secret value" action. + let unsecureNeedsLoad = $derived(edit && deployedSecret && variable.is_secret && !hasStagedValue) + const MAX_VARIABLE_LENGTH = 10000 let editorKind: 'plain' | 'json' | 'yaml' = $state('plain') @@ -91,16 +101,28 @@ /> -